RO-VIM-openstack/osm_rovim_openstack/vimconn_openstack.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 ##
  20
  21 """
  22 osconnector implements all the methods to interact with openstack using the python-neutronclient.
  23
  24 For the VNF forwarding graph, The OpenStack VIM connector calls the
  25 networking-sfc Neutron extension methods, whose resources are mapped
  26 to the VIM connector's SFC resources as follows:
  27 - Classification (OSM) -> Flow Classifier (Neutron)
  28 - Service Function Instance (OSM) -> Port Pair (Neutron)
  29 - Service Function (OSM) -> Port Pair Group (Neutron)
  30 - Service Function Path (OSM) -> Port Chain (Neutron)
  31 """
  32
  33 from osm_ro_plugin import vimconn
  34 # import json
  35 import logging
  36 import netaddr
  37 import time
  38 import yaml
  39 import random
  40 import re
  41 import copy
  42 from pprint import pformat
  43
  44 from novaclient import client as nClient, exceptions as nvExceptions
  45 from keystoneauth1.identity import v2, v3
  46 from keystoneauth1 import session
  47 import keystoneclient.exceptions as ksExceptions
  48 import keystoneclient.v3.client as ksClient_v3
  49 import keystoneclient.v2_0.client as ksClient_v2
  50 from glanceclient import client as glClient
  51 import glanceclient.exc as gl1Exceptions
  52 from cinderclient import client as cClient
  53 from http.client import HTTPException   # TODO py3 check that this base exception matches python2 httplib.HTTPException
  54 from neutronclient.neutron import client as neClient
  55 from neutronclient.common import exceptions as neExceptions
  56 from requests.exceptions import ConnectionError
  57
  58 __author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes, xFlow Research, Igor D.C., Eduardo Sousa"
  59 __date__ = "$22-sep-2017 23:59:59$"
  60
  61 """contain the openstack virtual machine status to openmano status"""
  62 vmStatus2manoFormat = {'ACTIVE': 'ACTIVE',
  63                        'PAUSED': 'PAUSED',
  64                        'SUSPENDED': 'SUSPENDED',
  65                        'SHUTOFF': 'INACTIVE',
  66                        'BUILD': 'BUILD',
  67                        'ERROR': 'ERROR',
  68                        'DELETED': 'DELETED'
  69                        }
  70 netStatus2manoFormat = {'ACTIVE': 'ACTIVE',
  71                         'PAUSED': 'PAUSED',
  72                         'INACTIVE': 'INACTIVE',
  73                         'BUILD': 'BUILD',
  74                         'ERROR': 'ERROR',
  75                         'DELETED': 'DELETED'
  76                         }
  77
  78 supportedClassificationTypes = ['legacy_flow_classifier']
  79
  80 # global var to have a timeout creating and deleting volumes
  81 volume_timeout = 1800
  82 server_timeout = 1800
  83
  84
  85 class SafeDumper(yaml.SafeDumper):
  86     def represent_data(self, data):
  87         # Openstack APIs use custom subclasses of dict and YAML safe dumper
  88         # is designed to not handle that (reference issue 142 of pyyaml)
  89         if isinstance(data, dict) and data.__class__ != dict:
  90             # A simple solution is to convert those items back to dicts
  91             data = dict(data.items())
  92
  93         return super(SafeDumper, self).represent_data(data)
  94
  95
  96 class vimconnector(vimconn.VimConnector):
  97     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None,
  98                  log_level=None, config={}, persistent_info={}):
  99         """using common constructor parameters. In this case
 100         'url' is the keystone authorization url,
 101         'url_admin' is not use
 102         """
 103         api_version = config.get('APIversion')
 104         if api_version and api_version not in ('v3.3', 'v2.0', '2', '3'):
 105             raise vimconn.VimConnException("Invalid value '{}' for config:APIversion. "
 106                                            "Allowed values are 'v3.3', 'v2.0', '2' or '3'".format(api_version))
 107         vim_type = config.get('vim_type')
 108         if vim_type and vim_type not in ('vio', 'VIO'):
 109             raise vimconn.VimConnException("Invalid value '{}' for config:vim_type."
 110                                            "Allowed values are 'vio' or 'VIO'".format(vim_type))
 111
 112         if config.get('dataplane_net_vlan_range') is not None:
 113             # validate vlan ranges provided by user
 114             self._validate_vlan_ranges(config.get('dataplane_net_vlan_range'), 'dataplane_net_vlan_range')
 115
 116         if config.get('multisegment_vlan_range') is not None:
 117             # validate vlan ranges provided by user
 118             self._validate_vlan_ranges(config.get('multisegment_vlan_range'), 'multisegment_vlan_range')
 119
 120         vimconn.VimConnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level,
 121                                       config)
 122
 123         if self.config.get("insecure") and self.config.get("ca_cert"):
 124             raise vimconn.VimConnException("options insecure and ca_cert are mutually exclusive")
 125         self.verify = True
 126         if self.config.get("insecure"):
 127             self.verify = False
 128         if self.config.get("ca_cert"):
 129             self.verify = self.config.get("ca_cert")
 130
 131         if not url:
 132             raise TypeError('url param can not be NoneType')
 133         self.persistent_info = persistent_info
 134         self.availability_zone = persistent_info.get('availability_zone', None)
 135         self.session = persistent_info.get('session', {'reload_client': True})
 136         self.my_tenant_id = self.session.get('my_tenant_id')
 137         self.nova = self.session.get('nova')
 138         self.neutron = self.session.get('neutron')
 139         self.cinder = self.session.get('cinder')
 140         self.glance = self.session.get('glance')
 141         # self.glancev1 = self.session.get('glancev1')
 142         self.keystone = self.session.get('keystone')
 143         self.api_version3 = self.session.get('api_version3')
 144         self.vim_type = self.config.get("vim_type")
 145         if self.vim_type:
 146             self.vim_type = self.vim_type.upper()
 147         if self.config.get("use_internal_endpoint"):
 148             self.endpoint_type = "internalURL"
 149         else:
 150             self.endpoint_type = None
 151
 152         logging.getLogger('urllib3').setLevel(logging.WARNING)
 153         logging.getLogger('keystoneauth').setLevel(logging.WARNING)
 154         logging.getLogger('novaclient').setLevel(logging.WARNING)
 155         self.logger = logging.getLogger('openmano.vim.openstack')
 156
 157         # allow security_groups to be a list or a single string
 158         if isinstance(self.config.get('security_groups'), str):
 159             self.config['security_groups'] = [self.config['security_groups']]
 160         self.security_groups_id = None
 161
 162         # ###### VIO Specific Changes #########
 163         if self.vim_type == "VIO":
 164             self.logger = logging.getLogger('openmano.vim.vio')
 165
 166         if log_level:
 167             self.logger.setLevel(getattr(logging, log_level))
 168
 169     def __getitem__(self, index):
 170         """Get individuals parameters.
 171         Throw KeyError"""
 172         if index == 'project_domain_id':
 173             return self.config.get("project_domain_id")
 174         elif index == 'user_domain_id':
 175             return self.config.get("user_domain_id")
 176         else:
 177             return vimconn.VimConnector.__getitem__(self, index)
 178
 179     def __setitem__(self, index, value):
 180         """Set individuals parameters and it is marked as dirty so to force connection reload.
 181         Throw KeyError"""
 182         if index == 'project_domain_id':
 183             self.config["project_domain_id"] = value
 184         elif index == 'user_domain_id':
 185             self.config["user_domain_id"] = value
 186         else:
 187             vimconn.VimConnector.__setitem__(self, index, value)
 188         self.session['reload_client'] = True
 189
 190     def serialize(self, value):
 191         """Serialization of python basic types.
 192
 193         In the case value is not serializable a message will be logged and a
 194         simple representation of the data that cannot be converted back to
 195         python is returned.
 196         """
 197         if isinstance(value, str):
 198             return value
 199
 200         try:
 201             return yaml.dump(value, Dumper=SafeDumper,
 202                              default_flow_style=True, width=256)
 203         except yaml.representer.RepresenterError:
 204             self.logger.debug('The following entity cannot be serialized in YAML:\n\n%s\n\n', pformat(value),
 205                               exc_info=True)
 206             return str(value)
 207
 208     def _reload_connection(self):
 209         """Called before any operation, it check if credentials has changed
 210         Throw keystoneclient.apiclient.exceptions.AuthorizationFailure
 211         """
 212         # TODO control the timing and possible token timeout, but it seams that python client does this task for us :-)
 213         if self.session['reload_client']:
 214             if self.config.get('APIversion'):
 215                 self.api_version3 = self.config['APIversion'] == 'v3.3' or self.config['APIversion'] == '3'
 216             else:  # get from ending auth_url that end with v3 or with v2.0
 217                 self.api_version3 = self.url.endswith("/v3") or self.url.endswith("/v3/")
 218             self.session['api_version3'] = self.api_version3
 219             if self.api_version3:
 220                 if self.config.get('project_domain_id') or self.config.get('project_domain_name'):
 221                     project_domain_id_default = None
 222                 else:
 223                     project_domain_id_default = 'default'
 224                 if self.config.get('user_domain_id') or self.config.get('user_domain_name'):
 225                     user_domain_id_default = None
 226                 else:
 227                     user_domain_id_default = 'default'
 228                 auth = v3.Password(auth_url=self.url,
 229                                    username=self.user,
 230                                    password=self.passwd,
 231                                    project_name=self.tenant_name,
 232                                    project_id=self.tenant_id,
 233                                    project_domain_id=self.config.get('project_domain_id', project_domain_id_default),
 234                                    user_domain_id=self.config.get('user_domain_id', user_domain_id_default),
 235                                    project_domain_name=self.config.get('project_domain_name'),
 236                                    user_domain_name=self.config.get('user_domain_name'))
 237             else:
 238                 auth = v2.Password(auth_url=self.url,
 239                                    username=self.user,
 240                                    password=self.passwd,
 241                                    tenant_name=self.tenant_name,
 242                                    tenant_id=self.tenant_id)
 243             sess = session.Session(auth=auth, verify=self.verify)
 244             # addedd region_name to keystone, nova, neutron and cinder to support distributed cloud for Wind River
 245             # Titanium cloud and StarlingX
 246             region_name = self.config.get('region_name')
 247             if self.api_version3:
 248                 self.keystone = ksClient_v3.Client(session=sess, endpoint_type=self.endpoint_type,
 249                                                    region_name=region_name)
 250             else:
 251                 self.keystone = ksClient_v2.Client(session=sess, endpoint_type=self.endpoint_type)
 252             self.session['keystone'] = self.keystone
 253             # In order to enable microversion functionality an explicit microversion must be specified in 'config'.
 254             # This implementation approach is due to the warning message in
 255             # https://developer.openstack.org/api-guide/compute/microversions.html
 256             # where it is stated that microversion backwards compatibility is not guaranteed and clients should
 257             # always require an specific microversion.
 258             # To be able to use 'device role tagging' functionality define 'microversion: 2.32' in datacenter config
 259             version = self.config.get("microversion")
 260             if not version:
 261                 version = "2.1"
 262             # addedd region_name to keystone, nova, neutron and cinder to support distributed cloud for Wind River
 263             # Titanium cloud and StarlingX
 264             self.nova = self.session['nova'] = nClient.Client(str(version), session=sess,
 265                                                               endpoint_type=self.endpoint_type, region_name=region_name)
 266             self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess,
 267                                                                      endpoint_type=self.endpoint_type,
 268                                                                      region_name=region_name)
 269             self.cinder = self.session['cinder'] = cClient.Client(2, session=sess, endpoint_type=self.endpoint_type,
 270                                                                   region_name=region_name)
 271             try:
 272                 self.my_tenant_id = self.session['my_tenant_id'] = sess.get_project_id()
 273             except Exception:
 274                 self.logger.error("Cannot get project_id from session", exc_info=True)
 275             if self.endpoint_type == "internalURL":
 276                 glance_service_id = self.keystone.services.list(name="glance")[0].id
 277                 glance_endpoint = self.keystone.endpoints.list(glance_service_id, interface="internal")[0].url
 278             else:
 279                 glance_endpoint = None
 280             self.glance = self.session['glance'] = glClient.Client(2, session=sess, endpoint=glance_endpoint)
 281             # using version 1 of glance client in new_image()
 282             # self.glancev1 = self.session['glancev1'] = glClient.Client('1', session=sess,
 283             #                                                            endpoint=glance_endpoint)
 284             self.session['reload_client'] = False
 285             self.persistent_info['session'] = self.session
 286             # add availablity zone info inside  self.persistent_info
 287             self._set_availablity_zones()
 288             self.persistent_info['availability_zone'] = self.availability_zone
 289             self.security_groups_id = None  # force to get again security_groups_ids next time they are needed
 290
 291     def __net_os2mano(self, net_list_dict):
 292         """Transform the net openstack format to mano format
 293         net_list_dict can be a list of dict or a single dict"""
 294         if type(net_list_dict) is dict:
 295             net_list_ = (net_list_dict,)
 296         elif type(net_list_dict) is list:
 297             net_list_ = net_list_dict
 298         else:
 299             raise TypeError("param net_list_dict must be a list or a dictionary")
 300         for net in net_list_:
 301             if net.get('provider:network_type') == "vlan":
 302                 net['type'] = 'data'
 303             else:
 304                 net['type'] = 'bridge'
 305
 306     def __classification_os2mano(self, class_list_dict):
 307         """Transform the openstack format (Flow Classifier) to mano format
 308         (Classification) class_list_dict can be a list of dict or a single dict
 309         """
 310         if isinstance(class_list_dict, dict):
 311             class_list_ = [class_list_dict]
 312         elif isinstance(class_list_dict, list):
 313             class_list_ = class_list_dict
 314         else:
 315             raise TypeError("param class_list_dict must be a list or a dictionary")
 316         for classification in class_list_:
 317             id = classification.pop('id')
 318             name = classification.pop('name')
 319             description = classification.pop('description')
 320             project_id = classification.pop('project_id')
 321             tenant_id = classification.pop('tenant_id')
 322             original_classification = copy.deepcopy(classification)
 323             classification.clear()
 324             classification['ctype'] = 'legacy_flow_classifier'
 325             classification['definition'] = original_classification
 326             classification['id'] = id
 327             classification['name'] = name
 328             classification['description'] = description
 329             classification['project_id'] = project_id
 330             classification['tenant_id'] = tenant_id
 331
 332     def __sfi_os2mano(self, sfi_list_dict):
 333         """Transform the openstack format (Port Pair) to mano format (SFI)
 334         sfi_list_dict can be a list of dict or a single dict
 335         """
 336         if isinstance(sfi_list_dict, dict):
 337             sfi_list_ = [sfi_list_dict]
 338         elif isinstance(sfi_list_dict, list):
 339             sfi_list_ = sfi_list_dict
 340         else:
 341             raise TypeError(
 342                 "param sfi_list_dict must be a list or a dictionary")
 343         for sfi in sfi_list_:
 344             sfi['ingress_ports'] = []
 345             sfi['egress_ports'] = []
 346             if sfi.get('ingress'):
 347                 sfi['ingress_ports'].append(sfi['ingress'])
 348             if sfi.get('egress'):
 349                 sfi['egress_ports'].append(sfi['egress'])
 350             del sfi['ingress']
 351             del sfi['egress']
 352             params = sfi.get('service_function_parameters')
 353             sfc_encap = False
 354             if params:
 355                 correlation = params.get('correlation')
 356                 if correlation:
 357                     sfc_encap = True
 358             sfi['sfc_encap'] = sfc_encap
 359             del sfi['service_function_parameters']
 360
 361     def __sf_os2mano(self, sf_list_dict):
 362         """Transform the openstack format (Port Pair Group) to mano format (SF)
 363         sf_list_dict can be a list of dict or a single dict
 364         """
 365         if isinstance(sf_list_dict, dict):
 366             sf_list_ = [sf_list_dict]
 367         elif isinstance(sf_list_dict, list):
 368             sf_list_ = sf_list_dict
 369         else:
 370             raise TypeError(
 371                 "param sf_list_dict must be a list or a dictionary")
 372         for sf in sf_list_:
 373             del sf['port_pair_group_parameters']
 374             sf['sfis'] = sf['port_pairs']
 375             del sf['port_pairs']
 376
 377     def __sfp_os2mano(self, sfp_list_dict):
 378         """Transform the openstack format (Port Chain) to mano format (SFP)
 379         sfp_list_dict can be a list of dict or a single dict
 380         """
 381         if isinstance(sfp_list_dict, dict):
 382             sfp_list_ = [sfp_list_dict]
 383         elif isinstance(sfp_list_dict, list):
 384             sfp_list_ = sfp_list_dict
 385         else:
 386             raise TypeError(
 387                 "param sfp_list_dict must be a list or a dictionary")
 388         for sfp in sfp_list_:
 389             params = sfp.pop('chain_parameters')
 390             sfc_encap = False
 391             if params:
 392                 correlation = params.get('correlation')
 393                 if correlation:
 394                     sfc_encap = True
 395             sfp['sfc_encap'] = sfc_encap
 396             sfp['spi'] = sfp.pop('chain_id')
 397             sfp['classifications'] = sfp.pop('flow_classifiers')
 398             sfp['service_functions'] = sfp.pop('port_pair_groups')
 399
 400     # placeholder for now; read TODO note below
 401     def _validate_classification(self, type, definition):
 402         # only legacy_flow_classifier Type is supported at this point
 403         return True
 404         # TODO(igordcard): this method should be an abstract method of an
 405         # abstract Classification class to be implemented by the specific
 406         # Types. Also, abstract vimconnector should call the validation
 407         # method before the implemented VIM connectors are called.
 408
 409     def _format_exception(self, exception):
 410         """Transform a keystone, nova, neutron  exception into a vimconn exception discovering the cause"""
 411
 412         message_error = str(exception)
 413         tip = ""
 414
 415         if isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound, ksExceptions.NotFound,
 416                                   gl1Exceptions.HTTPNotFound)):
 417             raise vimconn.VimConnNotFoundException(type(exception).__name__ + ": " + message_error)
 418         elif isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError,
 419                                     ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed)):
 420             if type(exception).__name__ == "SSLError":
 421                 tip = " (maybe option 'insecure' must be added to the VIM)"
 422             raise vimconn.VimConnConnectionException("Invalid URL or credentials{}: {}".format(tip, message_error))
 423         elif isinstance(exception, (KeyError, nvExceptions.BadRequest, ksExceptions.BadRequest)):
 424             raise vimconn.VimConnException(type(exception).__name__ + ": " + message_error)
 425         elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException,
 426                                     neExceptions.NeutronException)):
 427             raise vimconn.VimConnUnexpectedResponse(type(exception).__name__ + ": " + message_error)
 428         elif isinstance(exception, nvExceptions.Conflict):
 429             raise vimconn.VimConnConflictException(type(exception).__name__ + ": " + message_error)
 430         elif isinstance(exception, vimconn.VimConnException):
 431             raise exception
 432         else:  # ()
 433             self.logger.error("General Exception " + message_error, exc_info=True)
 434             raise vimconn.VimConnConnectionException(type(exception).__name__ + ": " + message_error)
 435
 436     def _get_ids_from_name(self):
 437         """
 438          Obtain ids from name of tenant and security_groups. Store at self .security_groups_id"
 439         :return: None
 440         """
 441         # get tenant_id if only tenant_name is supplied
 442         self._reload_connection()
 443         if not self.my_tenant_id:
 444             raise vimconn.VimConnConnectionException("Error getting tenant information from name={} id={}".
 445                                                      format(self.tenant_name, self.tenant_id))
 446         if self.config.get('security_groups') and not self.security_groups_id:
 447             # convert from name to id
 448             neutron_sg_list = self.neutron.list_security_groups(tenant_id=self.my_tenant_id)["security_groups"]
 449
 450             self.security_groups_id = []
 451             for sg in self.config.get('security_groups'):
 452                 for neutron_sg in neutron_sg_list:
 453                     if sg in (neutron_sg["id"], neutron_sg["name"]):
 454                         self.security_groups_id.append(neutron_sg["id"])
 455                         break
 456                 else:
 457                     self.security_groups_id = None
 458                     raise vimconn.VimConnConnectionException("Not found security group {} for this tenant".format(sg))
 459
 460     def check_vim_connectivity(self):
 461         # just get network list to check connectivity and credentials
 462         self.get_network_list(filter_dict={})
 463
 464     def get_tenant_list(self, filter_dict={}):
 465         """Obtain tenants of VIM
 466         filter_dict can contain the following keys:
 467             name: filter by tenant name
 468             id: filter by tenant uuid/id
 469             <other VIM specific>
 470         Returns the tenant list of dictionaries: [{'name':'<name>, 'id':'<id>, ...}, ...]
 471         """
 472         self.logger.debug("Getting tenants from VIM filter: '%s'", str(filter_dict))
 473         try:
 474             self._reload_connection()
 475             if self.api_version3:
 476                 project_class_list = self.keystone.projects.list(name=filter_dict.get("name"))
 477             else:
 478                 project_class_list = self.keystone.tenants.findall(**filter_dict)
 479             project_list = []
 480             for project in project_class_list:
 481                 if filter_dict.get('id') and filter_dict["id"] != project.id:
 482                     continue
 483                 project_list.append(project.to_dict())
 484             return project_list
 485         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError) as e:
 486             self._format_exception(e)
 487
 488     def new_tenant(self, tenant_name, tenant_description):
 489         """Adds a new tenant to openstack VIM. Returns the tenant identifier"""
 490         self.logger.debug("Adding a new tenant name: %s", tenant_name)
 491         try:
 492             self._reload_connection()
 493             if self.api_version3:
 494                 project = self.keystone.projects.create(tenant_name, self.config.get("project_domain_id", "default"),
 495                                                         description=tenant_description, is_domain=False)
 496             else:
 497                 project = self.keystone.tenants.create(tenant_name, tenant_description)
 498             return project.id
 499         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ksExceptions.BadRequest, ConnectionError)\
 500                 as e:
 501             self._format_exception(e)
 502
 503     def delete_tenant(self, tenant_id):
 504         """Delete a tenant from openstack VIM. Returns the old tenant identifier"""
 505         self.logger.debug("Deleting tenant %s from VIM", tenant_id)
 506         try:
 507             self._reload_connection()
 508             if self.api_version3:
 509                 self.keystone.projects.delete(tenant_id)
 510             else:
 511                 self.keystone.tenants.delete(tenant_id)
 512             return tenant_id
 513         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ksExceptions.NotFound, ConnectionError)\
 514                 as e:
 515             self._format_exception(e)
 516
 517     def new_network(self, net_name, net_type, ip_profile=None, shared=False, provider_network_profile=None):
 518         """Adds a tenant network to VIM
 519         Params:
 520             'net_name': name of the network
 521             'net_type': one of:
 522                 'bridge': overlay isolated network
 523                 'data':   underlay E-LAN network for Passthrough and SRIOV interfaces
 524                 'ptp':    underlay E-LINE network for Passthrough and SRIOV interfaces.
 525             'ip_profile': is a dict containing the IP parameters of the network
 526                 'ip_version': can be "IPv4" or "IPv6" (Currently only IPv4 is implemented)
 527                 'subnet_address': ip_prefix_schema, that is X.X.X.X/Y
 528                 'gateway_address': (Optional) ip_schema, that is X.X.X.X
 529                 'dns_address': (Optional) comma separated list of ip_schema, e.g. X.X.X.X[,X,X,X,X]
 530                 'dhcp_enabled': True or False
 531                 'dhcp_start_address': ip_schema, first IP to grant
 532                 'dhcp_count': number of IPs to grant.
 533             'shared': if this network can be seen/use by other tenants/organization
 534             'provider_network_profile': (optional) contains {segmentation-id: vlan, network-type: vlan|vxlan,
 535                                                              physical-network: physnet-label}
 536         Returns a tuple with the network identifier and created_items, or raises an exception on error
 537             created_items can be None or a dictionary where this method can include key-values that will be passed to
 538             the method delete_network. Can be used to store created segments, created l2gw connections, etc.
 539             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
 540             as not present.
 541         """
 542         self.logger.debug("Adding a new network to VIM name '%s', type '%s'", net_name, net_type)
 543         # self.logger.debug(">>>>>>>>>>>>>>>>>> IP profile %s", str(ip_profile))
 544
 545         try:
 546             vlan = None
 547             if provider_network_profile:
 548                 vlan = provider_network_profile.get("segmentation-id")
 549             new_net = None
 550             created_items = {}
 551             self._reload_connection()
 552             network_dict = {'name': net_name, 'admin_state_up': True}
 553             if net_type in ("data", "ptp"):
 554                 provider_physical_network = None
 555                 if provider_network_profile and provider_network_profile.get("physical-network"):
 556                     provider_physical_network = provider_network_profile.get("physical-network")
 557                     # provider-network must be one of the dataplane_physcial_netowrk if this is a list. If it is string
 558                     # or not declared, just ignore the checking
 559                     if isinstance(self.config.get('dataplane_physical_net'), (tuple, list)) and \
 560                             provider_physical_network not in self.config['dataplane_physical_net']:
 561                         raise vimconn.VimConnConflictException(
 562                             "Invalid parameter 'provider-network:physical-network' for network creation. '{}' is not "
 563                             "one of the declared list at VIM_config:dataplane_physical_net".format(
 564                                 provider_physical_network))
 565                 if not provider_physical_network:  # use the default dataplane_physical_net
 566                     provider_physical_network = self.config.get('dataplane_physical_net')
 567                     # if it is non empty list, use the first value. If it is a string use the value directly
 568                     if isinstance(provider_physical_network, (tuple, list)) and provider_physical_network:
 569                         provider_physical_network = provider_physical_network[0]
 570
 571                 if not provider_physical_network:
 572                     raise vimconn.VimConnConflictException(
 573                         "missing information needed for underlay networks. Provide 'dataplane_physical_net' "
 574                         "configuration at VIM or use the NS instantiation parameter 'provider-network.physical-network'"
 575                         " for the VLD")
 576
 577                 if not self.config.get('multisegment_support'):
 578                     network_dict["provider:physical_network"] = provider_physical_network
 579                     if provider_network_profile and "network-type" in provider_network_profile:
 580                         network_dict["provider:network_type"] = provider_network_profile["network-type"]
 581                     else:
 582                         network_dict["provider:network_type"] = self.config.get('dataplane_network_type', 'vlan')
 583                     if vlan:
 584                         network_dict["provider:segmentation_id"] = vlan
 585                 else:
 586                     # Multi-segment case
 587                     segment_list = []
 588                     segment1_dict = {
 589                         "provider:physical_network": '',
 590                         "provider:network_type": 'vxlan'
 591                     }
 592                     segment_list.append(segment1_dict)
 593                     segment2_dict = {
 594                         "provider:physical_network": provider_physical_network,
 595                         "provider:network_type": "vlan"
 596                     }
 597                     if vlan:
 598                         segment2_dict["provider:segmentation_id"] = vlan
 599                     elif self.config.get('multisegment_vlan_range'):
 600                         vlanID = self._generate_multisegment_vlanID()
 601                         segment2_dict["provider:segmentation_id"] = vlanID
 602                     # else
 603                     #     raise vimconn.VimConnConflictException(
 604                     #         "You must provide 'multisegment_vlan_range' at config dict before creating a multisegment
 605                     #         network")
 606                     segment_list.append(segment2_dict)
 607                     network_dict["segments"] = segment_list
 608
 609                 # VIO Specific Changes. It needs a concrete VLAN
 610                 if self.vim_type == "VIO" and vlan is None:
 611                     if self.config.get('dataplane_net_vlan_range') is None:
 612                         raise vimconn.VimConnConflictException(
 613                             "You must provide 'dataplane_net_vlan_range' in format [start_ID - end_ID] at VIM_config "
 614                             "for creating underlay networks")
 615                     network_dict["provider:segmentation_id"] = self._generate_vlanID()
 616
 617             network_dict["shared"] = shared
 618             if self.config.get("disable_network_port_security"):
 619                 network_dict["port_security_enabled"] = False
 620             new_net = self.neutron.create_network({'network': network_dict})
 621             # print new_net
 622             # create subnetwork, even if there is no profile
 623             if not ip_profile:
 624                 ip_profile = {}
 625             if not ip_profile.get('subnet_address'):
 626                 # Fake subnet is required
 627                 subnet_rand = random.randint(0, 255)
 628                 ip_profile['subnet_address'] = "192.168.{}.0/24".format(subnet_rand)
 629             if 'ip_version' not in ip_profile:
 630                 ip_profile['ip_version'] = "IPv4"
 631             subnet = {"name": net_name+"-subnet",
 632                       "network_id": new_net["network"]["id"],
 633                       "ip_version": 4 if ip_profile['ip_version'] == "IPv4" else 6,
 634                       "cidr": ip_profile['subnet_address']
 635                       }
 636             # Gateway should be set to None if not needed. Otherwise openstack assigns one by default
 637             if ip_profile.get('gateway_address'):
 638                 subnet['gateway_ip'] = ip_profile['gateway_address']
 639             else:
 640                 subnet['gateway_ip'] = None
 641             if ip_profile.get('dns_address'):
 642                 subnet['dns_nameservers'] = ip_profile['dns_address'].split(";")
 643             if 'dhcp_enabled' in ip_profile:
 644                 subnet['enable_dhcp'] = False if \
 645                     ip_profile['dhcp_enabled'] == "false" or ip_profile['dhcp_enabled'] is False else True
 646             if ip_profile.get('dhcp_start_address'):
 647                 subnet['allocation_pools'] = []
 648                 subnet['allocation_pools'].append(dict())
 649                 subnet['allocation_pools'][0]['start'] = ip_profile['dhcp_start_address']
 650             if ip_profile.get('dhcp_count'):
 651                 # parts = ip_profile['dhcp_start_address'].split('.')
 652                 # ip_int = (int(parts[0]) << 24) + (int(parts[1]) << 16) + (int(parts[2]) << 8) + int(parts[3])
 653                 ip_int = int(netaddr.IPAddress(ip_profile['dhcp_start_address']))
 654                 ip_int += ip_profile['dhcp_count'] - 1
 655                 ip_str = str(netaddr.IPAddress(ip_int))
 656                 subnet['allocation_pools'][0]['end'] = ip_str
 657             # self.logger.debug(">>>>>>>>>>>>>>>>>> Subnet: %s", str(subnet))
 658             self.neutron.create_subnet({"subnet": subnet})
 659
 660             if net_type == "data" and self.config.get('multisegment_support'):
 661                 if self.config.get('l2gw_support'):
 662                     l2gw_list = self.neutron.list_l2_gateways().get("l2_gateways", ())
 663                     for l2gw in l2gw_list:
 664                         l2gw_conn = {
 665                             "l2_gateway_id": l2gw["id"],
 666                             "network_id": new_net["network"]["id"],
 667                             "segmentation_id": str(vlanID),
 668                         }
 669                         new_l2gw_conn = self.neutron.create_l2_gateway_connection({"l2_gateway_connection": l2gw_conn})
 670                         created_items["l2gwconn:" + str(new_l2gw_conn["l2_gateway_connection"]["id"])] = True
 671             return new_net["network"]["id"], created_items
 672         except Exception as e:
 673             # delete l2gw connections (if any) before deleting the network
 674             for k, v in created_items.items():
 675                 if not v:  # skip already deleted
 676                     continue
 677                 try:
 678                     k_item, _, k_id = k.partition(":")
 679                     if k_item == "l2gwconn":
 680                         self.neutron.delete_l2_gateway_connection(k_id)
 681                 except Exception as e2:
 682                     self.logger.error("Error deleting l2 gateway connection: {}: {}".format(type(e2).__name__, e2))
 683             if new_net:
 684                 self.neutron.delete_network(new_net['network']['id'])
 685             self._format_exception(e)
 686
 687     def get_network_list(self, filter_dict={}):
 688         """Obtain tenant networks of VIM
 689         Filter_dict can be:
 690             name: network name
 691             id: network uuid
 692             shared: boolean
 693             tenant_id: tenant
 694             admin_state_up: boolean
 695             status: 'ACTIVE'
 696         Returns the network list of dictionaries
 697         """
 698         self.logger.debug("Getting network from VIM filter: '%s'", str(filter_dict))
 699         try:
 700             self._reload_connection()
 701             filter_dict_os = filter_dict.copy()
 702             if self.api_version3 and "tenant_id" in filter_dict_os:
 703                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')  # TODO check
 704             net_dict = self.neutron.list_networks(**filter_dict_os)
 705             net_list = net_dict["networks"]
 706             self.__net_os2mano(net_list)
 707             return net_list
 708         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException,
 709                 ConnectionError) as e:
 710             self._format_exception(e)
 711
 712     def get_network(self, net_id):
 713         """Obtain details of network from VIM
 714         Returns the network information from a network id"""
 715         self.logger.debug(" Getting tenant network %s from VIM", net_id)
 716         filter_dict = {"id": net_id}
 717         net_list = self.get_network_list(filter_dict)
 718         if len(net_list) == 0:
 719             raise vimconn.VimConnNotFoundException("Network '{}' not found".format(net_id))
 720         elif len(net_list) > 1:
 721             raise vimconn.VimConnConflictException("Found more than one network with this criteria")
 722         net = net_list[0]
 723         subnets = []
 724         for subnet_id in net.get("subnets", ()):
 725             try:
 726                 subnet = self.neutron.show_subnet(subnet_id)
 727             except Exception as e:
 728                 self.logger.error("osconnector.get_network(): Error getting subnet %s %s" % (net_id, str(e)))
 729                 subnet = {"id": subnet_id, "fault": str(e)}
 730             subnets.append(subnet)
 731         net["subnets"] = subnets
 732         net["encapsulation"] = net.get('provider:network_type')
 733         net["encapsulation_type"] = net.get('provider:network_type')
 734         net["segmentation_id"] = net.get('provider:segmentation_id')
 735         net["encapsulation_id"] = net.get('provider:segmentation_id')
 736         return net
 737
 738     def delete_network(self, net_id, created_items=None):
 739         """
 740         Removes a tenant network from VIM and its associated elements
 741         :param net_id: VIM identifier of the network, provided by method new_network
 742         :param created_items: dictionary with extra items to be deleted. provided by method new_network
 743         Returns the network identifier or raises an exception upon error or when network is not found
 744         """
 745         self.logger.debug("Deleting network '%s' from VIM", net_id)
 746         if created_items is None:
 747             created_items = {}
 748         try:
 749             self._reload_connection()
 750             # delete l2gw connections (if any) before deleting the network
 751             for k, v in created_items.items():
 752                 if not v:  # skip already deleted
 753                     continue
 754                 try:
 755                     k_item, _, k_id = k.partition(":")
 756                     if k_item == "l2gwconn":
 757                         self.neutron.delete_l2_gateway_connection(k_id)
 758                 except Exception as e:
 759                     self.logger.error("Error deleting l2 gateway connection: {}: {}".format(type(e).__name__, e))
 760             # delete VM ports attached to this networks before the network
 761             ports = self.neutron.list_ports(network_id=net_id)
 762             for p in ports['ports']:
 763                 try:
 764                     self.neutron.delete_port(p["id"])
 765                 except Exception as e:
 766                     self.logger.error("Error deleting port %s: %s", p["id"], str(e))
 767             self.neutron.delete_network(net_id)
 768             return net_id
 769         except (neExceptions.ConnectionFailed, neExceptions.NetworkNotFoundClient, neExceptions.NeutronException,
 770                 ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 771             self._format_exception(e)
 772
 773     def refresh_nets_status(self, net_list):
 774         """Get the status of the networks
 775            Params: the list of network identifiers
 776            Returns a dictionary with:
 777                 net_id:         #VIM id of this network
 778                     status:     #Mandatory. Text with one of:
 779                                 #  DELETED (not found at vim)
 780                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 781                                 #  OTHER (Vim reported other status not understood)
 782                                 #  ERROR (VIM indicates an ERROR status)
 783                                 #  ACTIVE, INACTIVE, DOWN (admin down),
 784                                 #  BUILD (on building process)
 785                                 #
 786                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 787                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 788
 789         """
 790         net_dict = {}
 791         for net_id in net_list:
 792             net = {}
 793             try:
 794                 net_vim = self.get_network(net_id)
 795                 if net_vim['status'] in netStatus2manoFormat:
 796                     net["status"] = netStatus2manoFormat[net_vim['status']]
 797                 else:
 798                     net["status"] = "OTHER"
 799                     net["error_msg"] = "VIM status reported " + net_vim['status']
 800
 801                 if net['status'] == "ACTIVE" and not net_vim['admin_state_up']:
 802                     net['status'] = 'DOWN'
 803
 804                 net['vim_info'] = self.serialize(net_vim)
 805
 806                 if net_vim.get('fault'):  # TODO
 807                     net['error_msg'] = str(net_vim['fault'])
 808             except vimconn.VimConnNotFoundException as e:
 809                 self.logger.error("Exception getting net status: %s", str(e))
 810                 net['status'] = "DELETED"
 811                 net['error_msg'] = str(e)
 812             except vimconn.VimConnException as e:
 813                 self.logger.error("Exception getting net status: %s", str(e))
 814                 net['status'] = "VIM_ERROR"
 815                 net['error_msg'] = str(e)
 816             net_dict[net_id] = net
 817         return net_dict
 818
 819     def get_flavor(self, flavor_id):
 820         """Obtain flavor details from the  VIM. Returns the flavor dict details"""
 821         self.logger.debug("Getting flavor '%s'", flavor_id)
 822         try:
 823             self._reload_connection()
 824             flavor = self.nova.flavors.find(id=flavor_id)
 825             # TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 826             return flavor.to_dict()
 827         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException,
 828                 ConnectionError) as e:
 829             self._format_exception(e)
 830
 831     def get_flavor_id_from_data(self, flavor_dict):
 832         """Obtain flavor id that match the flavor description
 833            Returns the flavor_id or raises a vimconnNotFoundException
 834            flavor_dict: contains the required ram, vcpus, disk
 835            If 'use_existing_flavors' is set to True at config, the closer flavor that provides same or more ram, vcpus
 836                 and disk is returned. Otherwise a flavor with exactly same ram, vcpus and disk is returned or a
 837                 vimconnNotFoundException is raised
 838         """
 839         exact_match = False if self.config.get('use_existing_flavors') else True
 840         try:
 841             self._reload_connection()
 842             flavor_candidate_id = None
 843             flavor_candidate_data = (10000, 10000, 10000)
 844             flavor_target = (flavor_dict["ram"], flavor_dict["vcpus"], flavor_dict["disk"])
 845             # numa=None
 846             extended = flavor_dict.get("extended", {})
 847             if extended:
 848                 # TODO
 849                 raise vimconn.VimConnNotFoundException("Flavor with EPA still not implemented")
 850                 # if len(numas) > 1:
 851                 #     raise vimconn.VimConnNotFoundException("Cannot find any flavor with more than one numa")
 852                 # numa=numas[0]
 853                 # numas = extended.get("numas")
 854             for flavor in self.nova.flavors.list():
 855                 epa = flavor.get_keys()
 856                 if epa:
 857                     continue
 858                     # TODO
 859                 flavor_data = (flavor.ram, flavor.vcpus, flavor.disk)
 860                 if flavor_data == flavor_target:
 861                     return flavor.id
 862                 elif not exact_match and flavor_target < flavor_data < flavor_candidate_data:
 863                     flavor_candidate_id = flavor.id
 864                     flavor_candidate_data = flavor_data
 865             if not exact_match and flavor_candidate_id:
 866                 return flavor_candidate_id
 867             raise vimconn.VimConnNotFoundException("Cannot find any flavor matching '{}'".format(flavor_dict))
 868         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException,
 869                 ConnectionError) as e:
 870             self._format_exception(e)
 871
 872     def process_resource_quota(self, quota, prefix, extra_specs):
 873         """
 874         :param prefix:
 875         :param extra_specs:
 876         :return:
 877         """
 878         if 'limit' in quota:
 879             extra_specs["quota:" + prefix + "_limit"] = quota['limit']
 880         if 'reserve' in quota:
 881             extra_specs["quota:" + prefix + "_reservation"] = quota['reserve']
 882         if 'shares' in quota:
 883             extra_specs["quota:" + prefix + "_shares_level"] = "custom"
 884             extra_specs["quota:" + prefix + "_shares_share"] = quota['shares']
 885
 886     def new_flavor(self, flavor_data, change_name_if_used=True):
 887         """Adds a tenant flavor to openstack VIM
 888         if change_name_if_used is True, it will change name in case of conflict, because it is not supported name
 889          repetition
 890         Returns the flavor identifier
 891         """
 892         self.logger.debug("Adding flavor '%s'", str(flavor_data))
 893         retry = 0
 894         max_retries = 3
 895         name_suffix = 0
 896         try:
 897             name = flavor_data['name']
 898             while retry < max_retries:
 899                 retry += 1
 900                 try:
 901                     self._reload_connection()
 902                     if change_name_if_used:
 903                         # get used names
 904                         fl_names = []
 905                         fl = self.nova.flavors.list()
 906                         for f in fl:
 907                             fl_names.append(f.name)
 908                         while name in fl_names:
 909                             name_suffix += 1
 910                             name = flavor_data['name']+"-" + str(name_suffix)
 911
 912                     ram = flavor_data.get('ram', 64)
 913                     vcpus = flavor_data.get('vcpus', 1)
 914                     extra_specs = {}
 915
 916                     extended = flavor_data.get("extended")
 917                     if extended:
 918                         numas = extended.get("numas")
 919                         if numas:
 920                             numa_nodes = len(numas)
 921                             if numa_nodes > 1:
 922                                 return -1, "Can not add flavor with more than one numa"
 923                             extra_specs["hw:numa_nodes"] = str(numa_nodes)
 924                             extra_specs["hw:mem_page_size"] = "large"
 925                             extra_specs["hw:cpu_policy"] = "dedicated"
 926                             extra_specs["hw:numa_mempolicy"] = "strict"
 927                             if self.vim_type == "VIO":
 928                                 extra_specs["vmware:extra_config"] = '{"numa.nodeAffinity":"0"}'
 929                                 extra_specs["vmware:latency_sensitivity_level"] = "high"
 930                             for numa in numas:
 931                                 # overwrite ram and vcpus
 932                                 # check if key 'memory' is present in numa else use ram value at flavor
 933                                 if 'memory' in numa:
 934                                     ram = numa['memory']*1024
 935                                 # See for reference: https://specs.openstack.org/openstack/nova-specs/specs/mitaka/
 936                                 # implemented/virt-driver-cpu-thread-pinning.html
 937                                 extra_specs["hw:cpu_sockets"] = 1
 938                                 if 'paired-threads' in numa:
 939                                     vcpus = numa['paired-threads']*2
 940                                     # cpu_thread_policy "require" implies that the compute node must have an
 941                                     # STM architecture
 942                                     extra_specs["hw:cpu_thread_policy"] = "require"
 943                                     extra_specs["hw:cpu_policy"] = "dedicated"
 944                                 elif 'cores' in numa:
 945                                     vcpus = numa['cores']
 946                                     # cpu_thread_policy "prefer" implies that the host must not have an SMT
 947                                     # architecture, or a non-SMT architecture will be emulated
 948                                     extra_specs["hw:cpu_thread_policy"] = "isolate"
 949                                     extra_specs["hw:cpu_policy"] = "dedicated"
 950                                 elif 'threads' in numa:
 951                                     vcpus = numa['threads']
 952                                     # cpu_thread_policy "prefer" implies that the host may or may not have an SMT
 953                                     # architecture
 954                                     extra_specs["hw:cpu_thread_policy"] = "prefer"
 955                                     extra_specs["hw:cpu_policy"] = "dedicated"
 956                                 # for interface in numa.get("interfaces",() ):
 957                                 #     if interface["dedicated"]=="yes":
 958                                 #         raise vimconn.VimConnException("Passthrough interfaces are not supported
 959                                 #         for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable)
 960                                 #     #TODO, add the key 'pci_passthrough:alias"="<label at config>:<number ifaces>"'
 961                                 #      when a way to connect it is available
 962                         elif extended.get("cpu-quota"):
 963                             self.process_resource_quota(extended.get("cpu-quota"), "cpu", extra_specs)
 964                         if extended.get("mem-quota"):
 965                             self.process_resource_quota(extended.get("mem-quota"), "memory", extra_specs)
 966                         if extended.get("vif-quota"):
 967                             self.process_resource_quota(extended.get("vif-quota"), "vif", extra_specs)
 968                         if extended.get("disk-io-quota"):
 969                             self.process_resource_quota(extended.get("disk-io-quota"), "disk_io", extra_specs)
 970                     # create flavor
 971                     new_flavor = self.nova.flavors.create(name,
 972                                                           ram,
 973                                                           vcpus,
 974                                                           flavor_data.get('disk', 0),
 975                                                           is_public=flavor_data.get('is_public', True)
 976                                                           )
 977                     # add metadata
 978                     if extra_specs:
 979                         new_flavor.set_keys(extra_specs)
 980                     return new_flavor.id
 981                 except nvExceptions.Conflict as e:
 982                     if change_name_if_used and retry < max_retries:
 983                         continue
 984                     self._format_exception(e)
 985         # except nvExceptions.BadRequest as e:
 986         except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError, KeyError) as e:
 987             self._format_exception(e)
 988
 989     def delete_flavor(self, flavor_id):
 990         """Deletes a tenant flavor from openstack VIM. Returns the old flavor_id
 991         """
 992         try:
 993             self._reload_connection()
 994             self.nova.flavors.delete(flavor_id)
 995             return flavor_id
 996         # except nvExceptions.BadRequest as e:
 997         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException,
 998                 ConnectionError) as e:
 999             self._format_exception(e)
1000
1001     def new_image(self, image_dict):
1002         """
1003         Adds a tenant image to VIM. imge_dict is a dictionary with:
1004             name: name
1005             disk_format: qcow2, vhd, vmdk, raw (by default), ...
1006             location: path or URI
1007             public: "yes" or "no"
1008             metadata: metadata of the image
1009         Returns the image_id
1010         """
1011         retry = 0
1012         max_retries = 3
1013         while retry < max_retries:
1014             retry += 1
1015             try:
1016                 self._reload_connection()
1017                 # determine format  http://docs.openstack.org/developer/glance/formats.html
1018                 if "disk_format" in image_dict:
1019                     disk_format = image_dict["disk_format"]
1020                 else:  # autodiscover based on extension
1021                     if image_dict['location'].endswith(".qcow2"):
1022                         disk_format = "qcow2"
1023                     elif image_dict['location'].endswith(".vhd"):
1024                         disk_format = "vhd"
1025                     elif image_dict['location'].endswith(".vmdk"):
1026                         disk_format = "vmdk"
1027                     elif image_dict['location'].endswith(".vdi"):
1028                         disk_format = "vdi"
1029                     elif image_dict['location'].endswith(".iso"):
1030                         disk_format = "iso"
1031                     elif image_dict['location'].endswith(".aki"):
1032                         disk_format = "aki"
1033                     elif image_dict['location'].endswith(".ari"):
1034                         disk_format = "ari"
1035                     elif image_dict['location'].endswith(".ami"):
1036                         disk_format = "ami"
1037                     else:
1038                         disk_format = "raw"
1039                 self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location'])
1040                 if self.vim_type == "VIO":
1041                     container_format = "bare"
1042                     if 'container_format' in image_dict:
1043                         container_format = image_dict['container_format']
1044                     new_image = self.glance.images.create(name=image_dict['name'], container_format=container_format,
1045                                                           disk_format=disk_format)
1046                 else:
1047                     new_image = self.glance.images.create(name=image_dict['name'])
1048                 if image_dict['location'].startswith("http"):
1049                     # TODO there is not a method to direct download. It must be downloaded locally with requests
1050                     raise vimconn.VimConnNotImplemented("Cannot create image from URL")
1051                 else:  # local path
1052                     with open(image_dict['location']) as fimage:
1053                         self.glance.images.upload(new_image.id, fimage)
1054                         # new_image = self.glancev1.images.create(name=image_dict['name'], is_public=
1055                         #  image_dict.get('public',"yes")=="yes",
1056                         #    container_format="bare", data=fimage, disk_format=disk_format)
1057                 metadata_to_load = image_dict.get('metadata')
1058                 # TODO location is a reserved word for current openstack versions. fixed for VIO please check
1059                 #  for openstack
1060                 if self.vim_type == "VIO":
1061                     metadata_to_load['upload_location'] = image_dict['location']
1062                 else:
1063                     metadata_to_load['location'] = image_dict['location']
1064                 self.glance.images.update(new_image.id, **metadata_to_load)
1065                 return new_image.id
1066             except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e:
1067                 self._format_exception(e)
1068             except (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError, ConnectionError) as e:
1069                 if retry == max_retries:
1070                     continue
1071                 self._format_exception(e)
1072             except IOError as e:  # can not open the file
1073                 raise vimconn.VimConnConnectionException("{}: {} for {}".format(type(e).__name__, e,
1074                                                                                 image_dict['location']),
1075                                                          http_code=vimconn.HTTP_Bad_Request)
1076
1077     def delete_image(self, image_id):
1078         """Deletes a tenant image from openstack VIM. Returns the old id
1079         """
1080         try:
1081             self._reload_connection()
1082             self.glance.images.delete(image_id)
1083             return image_id
1084         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException,
1085                 gl1Exceptions.CommunicationError, gl1Exceptions.HTTPNotFound, ConnectionError) as e:  # TODO remove
1086             self._format_exception(e)
1087
1088     def get_image_id_from_path(self, path):
1089         """Get the image id from image path in the VIM database. Returns the image_id"""
1090         try:
1091             self._reload_connection()
1092             images = self.glance.images.list()
1093             for image in images:
1094                 if image.metadata.get("location") == path:
1095                     return image.id
1096             raise vimconn.VimConnNotFoundException("image with location '{}' not found".format(path))
1097         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError,
1098                 ConnectionError) as e:
1099             self._format_exception(e)
1100
1101     def get_image_list(self, filter_dict={}):
1102         """Obtain tenant images from VIM
1103         Filter_dict can be:
1104             id: image id
1105             name: image name
1106             checksum: image checksum
1107         Returns the image list of dictionaries:
1108             [{<the fields at Filter_dict plus some VIM specific>}, ...]
1109             List can be empty
1110         """
1111         self.logger.debug("Getting image list from VIM filter: '%s'", str(filter_dict))
1112         try:
1113             self._reload_connection()
1114             # filter_dict_os = filter_dict.copy()
1115             # First we filter by the available filter fields: name, id. The others are removed.
1116             image_list = self.glance.images.list()
1117             filtered_list = []
1118             for image in image_list:
1119                 try:
1120                     if filter_dict.get("name") and image["name"] != filter_dict["name"]:
1121                         continue
1122                     if filter_dict.get("id") and image["id"] != filter_dict["id"]:
1123                         continue
1124                     if filter_dict.get("checksum") and image["checksum"] != filter_dict["checksum"]:
1125                         continue
1126
1127                     filtered_list.append(image.copy())
1128                 except gl1Exceptions.HTTPNotFound:
1129                     pass
1130             return filtered_list
1131         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError,
1132                 ConnectionError) as e:
1133             self._format_exception(e)
1134
1135     def __wait_for_vm(self, vm_id, status):
1136         """wait until vm is in the desired status and return True.
1137         If the VM gets in ERROR status, return false.
1138         If the timeout is reached generate an exception"""
1139         elapsed_time = 0
1140         while elapsed_time < server_timeout:
1141             vm_status = self.nova.servers.get(vm_id).status
1142             if vm_status == status:
1143                 return True
1144             if vm_status == 'ERROR':
1145                 return False
1146             time.sleep(5)
1147             elapsed_time += 5
1148
1149         # if we exceeded the timeout rollback
1150         if elapsed_time >= server_timeout:
1151             raise vimconn.VimConnException('Timeout waiting for instance ' + vm_id + ' to get ' + status,
1152                                            http_code=vimconn.HTTP_Request_Timeout)
1153
1154     def _get_openstack_availablity_zones(self):
1155         """
1156         Get from openstack availability zones available
1157         :return:
1158         """
1159         try:
1160             openstack_availability_zone = self.nova.availability_zones.list()
1161             openstack_availability_zone = [str(zone.zoneName) for zone in openstack_availability_zone
1162                                            if zone.zoneName != 'internal']
1163             return openstack_availability_zone
1164         except Exception:
1165             return None
1166
1167     def _set_availablity_zones(self):
1168         """
1169         Set vim availablity zone
1170         :return:
1171         """
1172
1173         if 'availability_zone' in self.config:
1174             vim_availability_zones = self.config.get('availability_zone')
1175             if isinstance(vim_availability_zones, str):
1176                 self.availability_zone = [vim_availability_zones]
1177             elif isinstance(vim_availability_zones, list):
1178                 self.availability_zone = vim_availability_zones
1179         else:
1180             self.availability_zone = self._get_openstack_availablity_zones()
1181
1182     def _get_vm_availability_zone(self, availability_zone_index, availability_zone_list):
1183         """
1184         Return thge availability zone to be used by the created VM.
1185         :return: The VIM availability zone to be used or None
1186         """
1187         if availability_zone_index is None:
1188             if not self.config.get('availability_zone'):
1189                 return None
1190             elif isinstance(self.config.get('availability_zone'), str):
1191                 return self.config['availability_zone']
1192             else:
1193                 # TODO consider using a different parameter at config for default AV and AV list match
1194                 return self.config['availability_zone'][0]
1195
1196         vim_availability_zones = self.availability_zone
1197         # check if VIM offer enough availability zones describe in the VNFD
1198         if vim_availability_zones and len(availability_zone_list) <= len(vim_availability_zones):
1199             # check if all the names of NFV AV match VIM AV names
1200             match_by_index = False
1201             for av in availability_zone_list:
1202                 if av not in vim_availability_zones:
1203                     match_by_index = True
1204                     break
1205             if match_by_index:
1206                 return vim_availability_zones[availability_zone_index]
1207             else:
1208                 return availability_zone_list[availability_zone_index]
1209         else:
1210             raise vimconn.VimConnConflictException("No enough availability zones at VIM for this deployment")
1211
1212     def new_vminstance(self, name, description, start, image_id, flavor_id, net_list, cloud_config=None, disk_list=None,
1213                        availability_zone_index=None, availability_zone_list=None):
1214         """Adds a VM instance to VIM
1215         Params:
1216             start: indicates if VM must start or boot in pause mode. Ignored
1217             image_id,flavor_id: iamge and flavor uuid
1218             net_list: list of interfaces, each one is a dictionary with:
1219                 name:
1220                 net_id: network uuid to connect
1221                 vpci: virtual vcpi to assign, ignored because openstack lack #TODO
1222                 model: interface model, ignored #TODO
1223                 mac_address: used for  SR-IOV ifaces #TODO for other types
1224                 use: 'data', 'bridge',  'mgmt'
1225                 type: 'virtual', 'PCI-PASSTHROUGH'('PF'), 'SR-IOV'('VF'), 'VFnotShared'
1226                 vim_id: filled/added by this function
1227                 floating_ip: True/False (or it can be None)
1228             'cloud_config': (optional) dictionary with:
1229                 'key-pairs': (optional) list of strings with the public key to be inserted to the default user
1230                 'users': (optional) list of users to be inserted, each item is a dict with:
1231                     'name': (mandatory) user name,
1232                     'key-pairs': (optional) list of strings with the public key to be inserted to the user
1233                 'user-data': (optional) string is a text script to be passed directly to cloud-init
1234                 'config-files': (optional). List of files to be transferred. Each item is a dict with:
1235                     'dest': (mandatory) string with the destination absolute path
1236                     'encoding': (optional, by default text). Can be one of:
1237                         'b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64'
1238                     'content' (mandatory): string with the content of the file
1239                     'permissions': (optional) string with file permissions, typically octal notation '0644'
1240                     'owner': (optional) file owner, string with the format 'owner:group'
1241                 'boot-data-drive': boolean to indicate if user-data must be passed using a boot drive (hard disk)
1242             'disk_list': (optional) list with additional disks to the VM. Each item is a dict with:
1243                 'image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted
1244                 'size': (mandatory) string with the size of the disk in GB
1245                 'vim_id' (optional) should use this existing volume id
1246             availability_zone_index: Index of availability_zone_list to use for this this VM. None if not AV required
1247             availability_zone_list: list of availability zones given by user in the VNFD descriptor.  Ignore if
1248                 availability_zone_index is None
1249                 #TODO ip, security groups
1250         Returns a tuple with the instance identifier and created_items or raises an exception on error
1251             created_items can be None or a dictionary where this method can include key-values that will be passed to
1252             the method delete_vminstance and action_vminstance. Can be used to store created ports, volumes, etc.
1253             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
1254             as not present.
1255         """
1256         self.logger.debug("new_vminstance input: image='%s' flavor='%s' nics='%s'", image_id, flavor_id, str(net_list))
1257         try:
1258             server = None
1259             created_items = {}
1260             # metadata = {}
1261             net_list_vim = []
1262             external_network = []
1263             # ^list of external networks to be connected to instance, later on used to create floating_ip
1264             no_secured_ports = []   # List of port-is with port-security disabled
1265             self._reload_connection()
1266             # metadata_vpci = {}   # For a specific neutron plugin
1267             block_device_mapping = None
1268
1269             for net in net_list:
1270                 if not net.get("net_id"):   # skip non connected iface
1271                     continue
1272
1273                 port_dict = {
1274                     "network_id": net["net_id"],
1275                     "name": net.get("name"),
1276                     "admin_state_up": True
1277                 }
1278                 if self.config.get("security_groups") and net.get("port_security") is not False and \
1279                         not self.config.get("no_port_security_extension"):
1280                     if not self.security_groups_id:
1281                         self._get_ids_from_name()
1282                     port_dict["security_groups"] = self.security_groups_id
1283
1284                 if net["type"] == "virtual":
1285                     pass
1286                     # if "vpci" in net:
1287                     #     metadata_vpci[ net["net_id"] ] = [[ net["vpci"], "" ]]
1288                 elif net["type"] == "VF" or net["type"] == "SR-IOV":  # for VF
1289                     # if "vpci" in net:
1290                     #     if "VF" not in metadata_vpci:
1291                     #         metadata_vpci["VF"]=[]
1292                     #     metadata_vpci["VF"].append([ net["vpci"], "" ])
1293                     port_dict["binding:vnic_type"] = "direct"
1294                     # VIO specific Changes
1295                     if self.vim_type == "VIO":
1296                         # Need to create port with port_security_enabled = False and no-security-groups
1297                         port_dict["port_security_enabled"] = False
1298                         port_dict["provider_security_groups"] = []
1299                         port_dict["security_groups"] = []
1300                 else:   # For PT PCI-PASSTHROUGH
1301                     # if "vpci" in net:
1302                     #     if "PF" not in metadata_vpci:
1303                     #         metadata_vpci["PF"]=[]
1304                     #     metadata_vpci["PF"].append([ net["vpci"], "" ])
1305                     port_dict["binding:vnic_type"] = "direct-physical"
1306                 if not port_dict["name"]:
1307                     port_dict["name"] = name
1308                 if net.get("mac_address"):
1309                     port_dict["mac_address"] = net["mac_address"]
1310                 if net.get("ip_address"):
1311                     port_dict["fixed_ips"] = [{'ip_address': net["ip_address"]}]
1312                     # TODO add 'subnet_id': <subnet_id>
1313                 new_port = self.neutron.create_port({"port": port_dict})
1314                 created_items["port:" + str(new_port["port"]["id"])] = True
1315                 net["mac_adress"] = new_port["port"]["mac_address"]
1316                 net["vim_id"] = new_port["port"]["id"]
1317                 # if try to use a network without subnetwork, it will return a emtpy list
1318                 fixed_ips = new_port["port"].get("fixed_ips")
1319                 if fixed_ips:
1320                     net["ip"] = fixed_ips[0].get("ip_address")
1321                 else:
1322                     net["ip"] = None
1323
1324                 port = {"port-id": new_port["port"]["id"]}
1325                 if float(self.nova.api_version.get_string()) >= 2.32:
1326                     port["tag"] = new_port["port"]["name"]
1327                 net_list_vim.append(port)
1328
1329                 if net.get('floating_ip', False):
1330                     net['exit_on_floating_ip_error'] = True
1331                     external_network.append(net)
1332                 elif net['use'] == 'mgmt' and self.config.get('use_floating_ip'):
1333                     net['exit_on_floating_ip_error'] = False
1334                     external_network.append(net)
1335                     net['floating_ip'] = self.config.get('use_floating_ip')
1336
1337                 # If port security is disabled when the port has not yet been attached to the VM, then all vm traffic
1338                 # is dropped.
1339                 # As a workaround we wait until the VM is active and then disable the port-security
1340                 if net.get("port_security") is False and not self.config.get("no_port_security_extension"):
1341                     no_secured_ports.append(new_port["port"]["id"])
1342
1343             # if metadata_vpci:
1344             #     metadata = {"pci_assignement": json.dumps(metadata_vpci)}
1345             #     if len(metadata["pci_assignement"]) >255:
1346             #         #limit the metadata size
1347             #         #metadata["pci_assignement"] = metadata["pci_assignement"][0:255]
1348             #         self.logger.warn("Metadata deleted since it exceeds the expected length (255) ")
1349             #         metadata = {}
1350
1351             self.logger.debug("name '%s' image_id '%s'flavor_id '%s' net_list_vim '%s' description '%s'",
1352                               name, image_id, flavor_id, str(net_list_vim), description)
1353
1354             # cloud config
1355             config_drive, userdata = self._create_user_data(cloud_config)
1356
1357             # Create additional volumes in case these are present in disk_list
1358             base_disk_index = ord('b')
1359             if disk_list:
1360                 block_device_mapping = {}
1361                 for disk in disk_list:
1362                     if disk.get('vim_id'):
1363                         block_device_mapping['_vd' + chr(base_disk_index)] = disk['vim_id']
1364                     else:
1365                         if 'image_id' in disk:
1366                             volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1367                                                                 chr(base_disk_index), imageRef=disk['image_id'])
1368                         else:
1369                             volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1370                                                                 chr(base_disk_index))
1371                         created_items["volume:" + str(volume.id)] = True
1372                         block_device_mapping['_vd' + chr(base_disk_index)] = volume.id
1373                     base_disk_index += 1
1374
1375                 # Wait until created volumes are with status available
1376                 elapsed_time = 0
1377                 while elapsed_time < volume_timeout:
1378                     for created_item in created_items:
1379                         v, _, volume_id = created_item.partition(":")
1380                         if v == 'volume':
1381                             if self.cinder.volumes.get(volume_id).status != 'available':
1382                                 break
1383                     else:  # all ready: break from while
1384                         break
1385                     time.sleep(5)
1386                     elapsed_time += 5
1387                 # If we exceeded the timeout rollback
1388                 if elapsed_time >= volume_timeout:
1389                     raise vimconn.VimConnException('Timeout creating volumes for instance ' + name,
1390                                                    http_code=vimconn.HTTP_Request_Timeout)
1391             # get availability Zone
1392             vm_av_zone = self._get_vm_availability_zone(availability_zone_index, availability_zone_list)
1393
1394             self.logger.debug("nova.servers.create({}, {}, {}, nics={}, security_groups={}, "
1395                               "availability_zone={}, key_name={}, userdata={}, config_drive={}, "
1396                               "block_device_mapping={})".format(name, image_id, flavor_id, net_list_vim,
1397                                                                 self.config.get("security_groups"), vm_av_zone,
1398                                                                 self.config.get('keypair'), userdata, config_drive,
1399                                                                 block_device_mapping))
1400             server = self.nova.servers.create(name, image_id, flavor_id, nics=net_list_vim,
1401                                               security_groups=self.config.get("security_groups"),
1402                                               # TODO remove security_groups in future versions. Already at neutron port
1403                                               availability_zone=vm_av_zone,
1404                                               key_name=self.config.get('keypair'),
1405                                               userdata=userdata,
1406                                               config_drive=config_drive,
1407                                               block_device_mapping=block_device_mapping
1408                                               )  # , description=description)
1409
1410             vm_start_time = time.time()
1411             # Previously mentioned workaround to wait until the VM is active and then disable the port-security
1412             if no_secured_ports:
1413                 self.__wait_for_vm(server.id, 'ACTIVE')
1414
1415             for port_id in no_secured_ports:
1416                 try:
1417                     self.neutron.update_port(port_id,
1418                                              {"port": {"port_security_enabled": False, "security_groups": None}})
1419                 except Exception:
1420                     raise vimconn.VimConnException("It was not possible to disable port security for port {}".format(
1421                         port_id))
1422             # print "DONE :-)", server
1423
1424             # pool_id = None
1425             if external_network:
1426                 floating_ips = self.neutron.list_floatingips().get("floatingips", ())
1427             for floating_network in external_network:
1428                 try:
1429                     assigned = False
1430                     while not assigned:
1431                         if floating_ips:
1432                             ip = floating_ips.pop(0)
1433                             if ip.get("port_id", False) or ip.get('tenant_id') != server.tenant_id:
1434                                 continue
1435                             if isinstance(floating_network['floating_ip'], str):
1436                                 if ip.get("floating_network_id") != floating_network['floating_ip']:
1437                                     continue
1438                             free_floating_ip = ip["id"]
1439                         else:
1440                             if isinstance(floating_network['floating_ip'], str) and \
1441                                     floating_network['floating_ip'].lower() != "true":
1442                                 pool_id = floating_network['floating_ip']
1443                             else:
1444                                 # Find the external network
1445                                 external_nets = list()
1446                                 for net in self.neutron.list_networks()['networks']:
1447                                     if net['router:external']:
1448                                         external_nets.append(net)
1449
1450                                 if len(external_nets) == 0:
1451                                     raise vimconn.VimConnException(
1452                                         "Cannot create floating_ip automatically since no external network is present",
1453                                         http_code=vimconn.HTTP_Conflict)
1454                                 if len(external_nets) > 1:
1455                                     raise vimconn.VimConnException(
1456                                         "Cannot create floating_ip automatically since multiple external networks are"
1457                                         " present", http_code=vimconn.HTTP_Conflict)
1458
1459                                 pool_id = external_nets[0].get('id')
1460                             param = {'floatingip': {'floating_network_id': pool_id, 'tenant_id': server.tenant_id}}
1461                             try:
1462                                 # self.logger.debug("Creating floating IP")
1463                                 new_floating_ip = self.neutron.create_floatingip(param)
1464                                 free_floating_ip = new_floating_ip['floatingip']['id']
1465                             except Exception as e:
1466                                 raise vimconn.VimConnException(type(e).__name__ + ": Cannot create new floating_ip " +
1467                                                                str(e), http_code=vimconn.HTTP_Conflict)
1468
1469                         while not assigned:
1470                             try:
1471                                 # the vim_id key contains the neutron.port_id
1472                                 self.neutron.update_floatingip(free_floating_ip,
1473                                                                {"floatingip": {"port_id": floating_network["vim_id"]}})
1474                                 # Using nove is deprecated on nova client 10.0
1475                                 assigned = True
1476                             except Exception as e:
1477                                 # openstack need some time after VM creation to asign an IP. So retry if fails
1478                                 vm_status = self.nova.servers.get(server.id).status
1479                                 if vm_status != 'ACTIVE' and vm_status != 'ERROR':
1480                                     if time.time() - vm_start_time < server_timeout:
1481                                         time.sleep(5)
1482                                         continue
1483                                 raise vimconn.VimConnException(
1484                                     "Cannot create floating_ip: {} {}".format(type(e).__name__, e),
1485                                     http_code=vimconn.HTTP_Conflict)
1486
1487                 except Exception as e:
1488                     if not floating_network['exit_on_floating_ip_error']:
1489                         self.logger.warning("Cannot create floating_ip. %s", str(e))
1490                         continue
1491                     raise
1492
1493             return server.id, created_items
1494         # except nvExceptions.NotFound as e:
1495         #     error_value=-vimconn.HTTP_Not_Found
1496         #     error_text= "vm instance %s not found" % vm_id
1497         # except TypeError as e:
1498         #     raise vimconn.VimConnException(type(e).__name__ + ": "+  str(e), http_code=vimconn.HTTP_Bad_Request)
1499
1500         except Exception as e:
1501             server_id = None
1502             if server:
1503                 server_id = server.id
1504             try:
1505                 self.delete_vminstance(server_id, created_items)
1506             except Exception as e2:
1507                 self.logger.error("new_vminstance rollback fail {}".format(e2))
1508
1509             self._format_exception(e)
1510
1511     def get_vminstance(self, vm_id):
1512         """Returns the VM instance information from VIM"""
1513         # self.logger.debug("Getting VM from VIM")
1514         try:
1515             self._reload_connection()
1516             server = self.nova.servers.find(id=vm_id)
1517             # TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
1518             return server.to_dict()
1519         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound,
1520                 ConnectionError) as e:
1521             self._format_exception(e)
1522
1523     def get_vminstance_console(self, vm_id, console_type="vnc"):
1524         """
1525         Get a console for the virtual machine
1526         Params:
1527             vm_id: uuid of the VM
1528             console_type, can be:
1529                 "novnc" (by default), "xvpvnc" for VNC types,
1530                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
1531         Returns dict with the console parameters:
1532                 protocol: ssh, ftp, http, https, ...
1533                 server:   usually ip address
1534                 port:     the http, ssh, ... port
1535                 suffix:   extra text, e.g. the http path and query string
1536         """
1537         self.logger.debug("Getting VM CONSOLE from VIM")
1538         try:
1539             self._reload_connection()
1540             server = self.nova.servers.find(id=vm_id)
1541             if console_type is None or console_type == "novnc":
1542                 console_dict = server.get_vnc_console("novnc")
1543             elif console_type == "xvpvnc":
1544                 console_dict = server.get_vnc_console(console_type)
1545             elif console_type == "rdp-html5":
1546                 console_dict = server.get_rdp_console(console_type)
1547             elif console_type == "spice-html5":
1548                 console_dict = server.get_spice_console(console_type)
1549             else:
1550                 raise vimconn.VimConnException("console type '{}' not allowed".format(console_type),
1551                                                http_code=vimconn.HTTP_Bad_Request)
1552
1553             console_dict1 = console_dict.get("console")
1554             if console_dict1:
1555                 console_url = console_dict1.get("url")
1556                 if console_url:
1557                     # parse console_url
1558                     protocol_index = console_url.find("//")
1559                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1560                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1561                     if protocol_index < 0 or port_index < 0 or suffix_index < 0:
1562                         return -vimconn.HTTP_Internal_Server_Error, "Unexpected response from VIM"
1563                     console_dict = {"protocol": console_url[0:protocol_index],
1564                                     "server": console_url[protocol_index+2:port_index],
1565                                     "port": console_url[port_index:suffix_index],
1566                                     "suffix": console_url[suffix_index+1:]
1567                                     }
1568                     protocol_index += 2
1569                     return console_dict
1570             raise vimconn.VimConnUnexpectedResponse("Unexpected response from VIM")
1571
1572         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException,
1573                 nvExceptions.BadRequest, ConnectionError) as e:
1574             self._format_exception(e)
1575
1576     def delete_vminstance(self, vm_id, created_items=None):
1577         """Removes a VM instance from VIM. Returns the old identifier
1578         """
1579         # print "osconnector: Getting VM from VIM"
1580         if created_items is None:
1581             created_items = {}
1582         try:
1583             self._reload_connection()
1584             # delete VM ports attached to this networks before the virtual machine
1585             for k, v in created_items.items():
1586                 if not v:  # skip already deleted
1587                     continue
1588                 try:
1589                     k_item, _, k_id = k.partition(":")
1590                     if k_item == "port":
1591                         self.neutron.delete_port(k_id)
1592                 except Exception as e:
1593                     self.logger.error("Error deleting port: {}: {}".format(type(e).__name__, e))
1594
1595             # #commented because detaching the volumes makes the servers.delete not work properly ?!?
1596             # #dettach volumes attached
1597             # server = self.nova.servers.get(vm_id)
1598             # volumes_attached_dict = server._info['os-extended-volumes:volumes_attached']   #volume['id']
1599             # #for volume in volumes_attached_dict:
1600             # #    self.cinder.volumes.detach(volume['id'])
1601
1602             if vm_id:
1603                 self.nova.servers.delete(vm_id)
1604
1605             # delete volumes. Although having detached, they should have in active status before deleting
1606             # we ensure in this loop
1607             keep_waiting = True
1608             elapsed_time = 0
1609             while keep_waiting and elapsed_time < volume_timeout:
1610                 keep_waiting = False
1611                 for k, v in created_items.items():
1612                     if not v:  # skip already deleted
1613                         continue
1614                     try:
1615                         k_item, _, k_id = k.partition(":")
1616                         if k_item == "volume":
1617                             if self.cinder.volumes.get(k_id).status != 'available':
1618                                 keep_waiting = True
1619                             else:
1620                                 self.cinder.volumes.delete(k_id)
1621                     except Exception as e:
1622                         self.logger.error("Error deleting volume: {}: {}".format(type(e).__name__, e))
1623                 if keep_waiting:
1624                     time.sleep(1)
1625                     elapsed_time += 1
1626             return None
1627         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException,
1628                 ConnectionError) as e:
1629             self._format_exception(e)
1630
1631     def refresh_vms_status(self, vm_list):
1632         """Get the status of the virtual machines and their interfaces/ports
1633            Params: the list of VM identifiers
1634            Returns a dictionary with:
1635                 vm_id:          #VIM id of this Virtual Machine
1636                     status:     #Mandatory. Text with one of:
1637                                 #  DELETED (not found at vim)
1638                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1639                                 #  OTHER (Vim reported other status not understood)
1640                                 #  ERROR (VIM indicates an ERROR status)
1641                                 #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
1642                                 #  CREATING (on building process), ERROR
1643                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
1644                                 #
1645                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
1646                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
1647                     interfaces:
1648                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
1649                         mac_address:      #Text format XX:XX:XX:XX:XX:XX
1650                         vim_net_id:       #network id where this interface is connected
1651                         vim_interface_id: #interface/port VIM id
1652                         ip_address:       #null, or text with IPv4, IPv6 address
1653                         compute_node:     #identification of compute node where PF,VF interface is allocated
1654                         pci:              #PCI address of the NIC that hosts the PF,VF
1655                         vlan:             #physical VLAN used for VF
1656         """
1657         vm_dict = {}
1658         self.logger.debug("refresh_vms status: Getting tenant VM instance information from VIM")
1659         for vm_id in vm_list:
1660             vm = {}
1661             try:
1662                 vm_vim = self.get_vminstance(vm_id)
1663                 if vm_vim['status'] in vmStatus2manoFormat:
1664                     vm['status'] = vmStatus2manoFormat[vm_vim['status']]
1665                 else:
1666                     vm['status'] = "OTHER"
1667                     vm['error_msg'] = "VIM status reported " + vm_vim['status']
1668
1669                 vm['vim_info'] = self.serialize(vm_vim)
1670
1671                 vm["interfaces"] = []
1672                 if vm_vim.get('fault'):
1673                     vm['error_msg'] = str(vm_vim['fault'])
1674                 # get interfaces
1675                 try:
1676                     self._reload_connection()
1677                     port_dict = self.neutron.list_ports(device_id=vm_id)
1678                     for port in port_dict["ports"]:
1679                         interface = {}
1680                         interface['vim_info'] = self.serialize(port)
1681                         interface["mac_address"] = port.get("mac_address")
1682                         interface["vim_net_id"] = port["network_id"]
1683                         interface["vim_interface_id"] = port["id"]
1684                         # check if OS-EXT-SRV-ATTR:host is there,
1685                         # in case of non-admin credentials, it will be missing
1686                         if vm_vim.get('OS-EXT-SRV-ATTR:host'):
1687                             interface["compute_node"] = vm_vim['OS-EXT-SRV-ATTR:host']
1688                         interface["pci"] = None
1689
1690                         # check if binding:profile is there,
1691                         # in case of non-admin credentials, it will be missing
1692                         if port.get('binding:profile'):
1693                             if port['binding:profile'].get('pci_slot'):
1694                                 # TODO: At the moment sr-iov pci addresses are converted to PF pci addresses by setting
1695                                 #  the slot to 0x00
1696                                 # TODO: This is just a workaround valid for niantinc. Find a better way to do so
1697                                 #   CHANGE DDDD:BB:SS.F to DDDD:BB:00.(F%2)   assuming there are 2 ports per nic
1698                                 pci = port['binding:profile']['pci_slot']
1699                                 # interface["pci"] = pci[:-4] + "00." + str(int(pci[-1]) % 2)
1700                                 interface["pci"] = pci
1701                         interface["vlan"] = None
1702                         if port.get('binding:vif_details'):
1703                             interface["vlan"] = port['binding:vif_details'].get('vlan')
1704                         # Get vlan from network in case not present in port for those old openstacks and cases where
1705                         # it is needed vlan at PT
1706                         if not interface["vlan"]:
1707                             # if network is of type vlan and port is of type direct (sr-iov) then set vlan id
1708                             network = self.neutron.show_network(port["network_id"])
1709                             if network['network'].get('provider:network_type') == 'vlan':
1710                                 # and port.get("binding:vnic_type") in ("direct", "direct-physical"):
1711                                 interface["vlan"] = network['network'].get('provider:segmentation_id')
1712                         ips = []
1713                         # look for floating ip address
1714                         try:
1715                             floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
1716                             if floating_ip_dict.get("floatingips"):
1717                                 ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address"))
1718                         except Exception:
1719                             pass
1720
1721                         for subnet in port["fixed_ips"]:
1722                             ips.append(subnet["ip_address"])
1723                         interface["ip_address"] = ";".join(ips)
1724                         vm["interfaces"].append(interface)
1725                 except Exception as e:
1726                     self.logger.error("Error getting vm interface information {}: {}".format(type(e).__name__, e),
1727                                       exc_info=True)
1728             except vimconn.VimConnNotFoundException as e:
1729                 self.logger.error("Exception getting vm status: %s", str(e))
1730                 vm['status'] = "DELETED"
1731                 vm['error_msg'] = str(e)
1732             except vimconn.VimConnException as e:
1733                 self.logger.error("Exception getting vm status: %s", str(e))
1734                 vm['status'] = "VIM_ERROR"
1735                 vm['error_msg'] = str(e)
1736             vm_dict[vm_id] = vm
1737         return vm_dict
1738
1739     def action_vminstance(self, vm_id, action_dict, created_items={}):
1740         """Send and action over a VM instance from VIM
1741         Returns None or the console dict if the action was successfully sent to the VIM"""
1742         self.logger.debug("Action over VM '%s': %s", vm_id, str(action_dict))
1743         try:
1744             self._reload_connection()
1745             server = self.nova.servers.find(id=vm_id)
1746             if "start" in action_dict:
1747                 if action_dict["start"] == "rebuild":
1748                     server.rebuild()
1749                 else:
1750                     if server.status == "PAUSED":
1751                         server.unpause()
1752                     elif server.status == "SUSPENDED":
1753                         server.resume()
1754                     elif server.status == "SHUTOFF":
1755                         server.start()
1756             elif "pause" in action_dict:
1757                 server.pause()
1758             elif "resume" in action_dict:
1759                 server.resume()
1760             elif "shutoff" in action_dict or "shutdown" in action_dict:
1761                 server.stop()
1762             elif "forceOff" in action_dict:
1763                 server.stop()  # TODO
1764             elif "terminate" in action_dict:
1765                 server.delete()
1766             elif "createImage" in action_dict:
1767                 server.create_image()
1768                 # "path":path_schema,
1769                 # "description":description_schema,
1770                 # "name":name_schema,
1771                 # "metadata":metadata_schema,
1772                 # "imageRef": id_schema,
1773                 # "disk": {"oneOf":[{"type": "null"}, {"type":"string"}] },
1774             elif "rebuild" in action_dict:
1775                 server.rebuild(server.image['id'])
1776             elif "reboot" in action_dict:
1777                 server.reboot()   # reboot_type='SOFT'
1778             elif "console" in action_dict:
1779                 console_type = action_dict["console"]
1780                 if console_type is None or console_type == "novnc":
1781                     console_dict = server.get_vnc_console("novnc")
1782                 elif console_type == "xvpvnc":
1783                     console_dict = server.get_vnc_console(console_type)
1784                 elif console_type == "rdp-html5":
1785                     console_dict = server.get_rdp_console(console_type)
1786                 elif console_type == "spice-html5":
1787                     console_dict = server.get_spice_console(console_type)
1788                 else:
1789                     raise vimconn.VimConnException("console type '{}' not allowed".format(console_type),
1790                                                    http_code=vimconn.HTTP_Bad_Request)
1791                 try:
1792                     console_url = console_dict["console"]["url"]
1793                     # parse console_url
1794                     protocol_index = console_url.find("//")
1795                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1796                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1797                     if protocol_index < 0 or port_index < 0 or suffix_index < 0:
1798                         raise vimconn.VimConnException("Unexpected response from VIM " + str(console_dict))
1799                     console_dict2 = {"protocol": console_url[0:protocol_index],
1800                                      "server": console_url[protocol_index+2: port_index],
1801                                      "port": int(console_url[port_index+1: suffix_index]),
1802                                      "suffix": console_url[suffix_index+1:]
1803                                      }
1804                     return console_dict2
1805                 except Exception:
1806                     raise vimconn.VimConnException("Unexpected response from VIM " + str(console_dict))
1807
1808             return None
1809         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound,
1810                 ConnectionError) as e:
1811             self._format_exception(e)
1812         # TODO insert exception vimconn.HTTP_Unauthorized
1813
1814     # ###### VIO Specific Changes #########
1815     def _generate_vlanID(self):
1816         """
1817          Method to get unused vlanID
1818             Args:
1819                 None
1820             Returns:
1821                 vlanID
1822         """
1823         # Get used VLAN IDs
1824         usedVlanIDs = []
1825         networks = self.get_network_list()
1826         for net in networks:
1827             if net.get('provider:segmentation_id'):
1828                 usedVlanIDs.append(net.get('provider:segmentation_id'))
1829         used_vlanIDs = set(usedVlanIDs)
1830
1831         # find unused VLAN ID
1832         for vlanID_range in self.config.get('dataplane_net_vlan_range'):
1833             try:
1834                 start_vlanid, end_vlanid = map(int, vlanID_range.replace(" ", "").split("-"))
1835                 for vlanID in range(start_vlanid, end_vlanid + 1):
1836                     if vlanID not in used_vlanIDs:
1837                         return vlanID
1838             except Exception as exp:
1839                 raise vimconn.VimConnException("Exception {} occurred while generating VLAN ID.".format(exp))
1840         else:
1841             raise vimconn.VimConnConflictException(
1842                 "Unable to create the SRIOV VLAN network. All given Vlan IDs {} are in use.".format(
1843                     self.config.get('dataplane_net_vlan_range')))
1844
1845     def _generate_multisegment_vlanID(self):
1846         """
1847          Method to get unused vlanID
1848             Args:
1849                 None
1850             Returns:
1851                 vlanID
1852         """
1853         # Get used VLAN IDs
1854         usedVlanIDs = []
1855         networks = self.get_network_list()
1856         for net in networks:
1857             if net.get('provider:network_type') == "vlan" and net.get('provider:segmentation_id'):
1858                 usedVlanIDs.append(net.get('provider:segmentation_id'))
1859             elif net.get('segments'):
1860                 for segment in net.get('segments'):
1861                     if segment.get('provider:network_type') == "vlan" and segment.get('provider:segmentation_id'):
1862                         usedVlanIDs.append(segment.get('provider:segmentation_id'))
1863         used_vlanIDs = set(usedVlanIDs)
1864
1865         # find unused VLAN ID
1866         for vlanID_range in self.config.get('multisegment_vlan_range'):
1867             try:
1868                 start_vlanid, end_vlanid = map(int, vlanID_range.replace(" ", "").split("-"))
1869                 for vlanID in range(start_vlanid, end_vlanid + 1):
1870                     if vlanID not in used_vlanIDs:
1871                         return vlanID
1872             except Exception as exp:
1873                 raise vimconn.VimConnException("Exception {} occurred while generating VLAN ID.".format(exp))
1874         else:
1875             raise vimconn.VimConnConflictException(
1876                 "Unable to create the VLAN segment. All VLAN IDs {} are in use.".format(
1877                     self.config.get('multisegment_vlan_range')))
1878
1879     def _validate_vlan_ranges(self, input_vlan_range, text_vlan_range):
1880         """
1881         Method to validate user given vlanID ranges
1882             Args:  None
1883             Returns: None
1884         """
1885         for vlanID_range in input_vlan_range:
1886             vlan_range = vlanID_range.replace(" ", "")
1887             # validate format
1888             vlanID_pattern = r'(\d)*-(\d)*$'
1889             match_obj = re.match(vlanID_pattern, vlan_range)
1890             if not match_obj:
1891                 raise vimconn.VimConnConflictException(
1892                     "Invalid VLAN range for {}: {}.You must provide '{}' in format [start_ID - end_ID].".format(
1893                         text_vlan_range, vlanID_range, text_vlan_range))
1894
1895             start_vlanid, end_vlanid = map(int, vlan_range.split("-"))
1896             if start_vlanid <= 0:
1897                 raise vimconn.VimConnConflictException(
1898                     "Invalid VLAN range for {}: {}. Start ID can not be zero. For VLAN "
1899                     "networks valid IDs are 1 to 4094 ".format(text_vlan_range, vlanID_range))
1900             if end_vlanid > 4094:
1901                 raise vimconn.VimConnConflictException(
1902                     "Invalid VLAN range for {}: {}. End VLAN ID can not be greater than 4094. For VLAN "
1903                     "networks valid IDs are 1 to 4094 ".format(text_vlan_range, vlanID_range))
1904
1905             if start_vlanid > end_vlanid:
1906                 raise vimconn.VimConnConflictException(
1907                     "Invalid VLAN range for {}: {}. You must provide '{}' in format start_ID - end_ID and "
1908                     "start_ID < end_ID ".format(text_vlan_range, vlanID_range, text_vlan_range))
1909
1910     # NOT USED FUNCTIONS
1911
1912     def new_external_port(self, port_data):
1913         """Adds a external port to VIM
1914            Returns the port identifier"""
1915         # TODO openstack if needed
1916         return -vimconn.HTTP_Internal_Server_Error, "osconnector.new_external_port() not implemented"
1917
1918     def connect_port_network(self, port_id, network_id, admin=False):
1919         """Connects a external port to a network
1920             Returns status code of the VIM response"""
1921         # TODO openstack if needed
1922         return -vimconn.HTTP_Internal_Server_Error, "osconnector.connect_port_network() not implemented"
1923
1924     def new_user(self, user_name, user_passwd, tenant_id=None):
1925         """Adds a new user to openstack VIM
1926            Returns the user identifier"""
1927         self.logger.debug("osconnector: Adding a new user to VIM")
1928         try:
1929             self._reload_connection()
1930             user = self.keystone.users.create(user_name, password=user_passwd, default_project=tenant_id)
1931             # self.keystone.tenants.add_user(self.k_creds["username"], #role)
1932             return user.id
1933         except ksExceptions.ConnectionError as e:
1934             error_value = -vimconn.HTTP_Bad_Request
1935             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
1936         except ksExceptions.ClientException as e:  # TODO remove
1937             error_value = -vimconn.HTTP_Bad_Request
1938             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
1939         # TODO insert exception vimconn.HTTP_Unauthorized
1940         # if reaching here is because an exception
1941         self.logger.debug("new_user " + error_text)
1942         return error_value, error_text
1943
1944     def delete_user(self, user_id):
1945         """Delete a user from openstack VIM
1946            Returns the user identifier"""
1947         if self.debug:
1948             print("osconnector: Deleting  a  user from VIM")
1949         try:
1950             self._reload_connection()
1951             self.keystone.users.delete(user_id)
1952             return 1, user_id
1953         except ksExceptions.ConnectionError as e:
1954             error_value = -vimconn.HTTP_Bad_Request
1955             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
1956         except ksExceptions.NotFound as e:
1957             error_value = -vimconn.HTTP_Not_Found
1958             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
1959         except ksExceptions.ClientException as e:  # TODO remove
1960             error_value = -vimconn.HTTP_Bad_Request
1961             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
1962         # TODO insert exception vimconn.HTTP_Unauthorized
1963         # if reaching here is because an exception
1964         self.logger.debug("delete_tenant " + error_text)
1965         return error_value, error_text
1966
1967     def get_hosts_info(self):
1968         """Get the information of deployed hosts
1969         Returns the hosts content"""
1970         if self.debug:
1971             print("osconnector: Getting Host info from VIM")
1972         try:
1973             h_list = []
1974             self._reload_connection()
1975             hypervisors = self.nova.hypervisors.list()
1976             for hype in hypervisors:
1977                 h_list.append(hype.to_dict())
1978             return 1, {"hosts": h_list}
1979         except nvExceptions.NotFound as e:
1980             error_value = -vimconn.HTTP_Not_Found
1981             error_text = (str(e) if len(e.args) == 0 else str(e.args[0]))
1982         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1983             error_value = -vimconn.HTTP_Bad_Request
1984             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
1985         # TODO insert exception vimconn.HTTP_Unauthorized
1986         # if reaching here is because an exception
1987         self.logger.debug("get_hosts_info " + error_text)
1988         return error_value, error_text
1989
1990     def get_hosts(self, vim_tenant):
1991         """Get the hosts and deployed instances
1992         Returns the hosts content"""
1993         r, hype_dict = self.get_hosts_info()
1994         if r < 0:
1995             return r, hype_dict
1996         hypervisors = hype_dict["hosts"]
1997         try:
1998             servers = self.nova.servers.list()
1999             for hype in hypervisors:
2000                 for server in servers:
2001                     if server.to_dict()['OS-EXT-SRV-ATTR:hypervisor_hostname'] == hype['hypervisor_hostname']:
2002                         if 'vm' in hype:
2003                             hype['vm'].append(server.id)
2004                         else:
2005                             hype['vm'] = [server.id]
2006             return 1, hype_dict
2007         except nvExceptions.NotFound as e:
2008             error_value = -vimconn.HTTP_Not_Found
2009             error_text = (str(e) if len(e.args) == 0 else str(e.args[0]))
2010         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
2011             error_value = -vimconn.HTTP_Bad_Request
2012             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
2013         # TODO insert exception vimconn.HTTP_Unauthorized
2014         # if reaching here is because an exception
2015         self.logger.debug("get_hosts " + error_text)
2016         return error_value, error_text
2017
2018     def new_classification(self, name, ctype, definition):
2019         self.logger.debug('Adding a new (Traffic) Classification to VIM, named %s', name)
2020         try:
2021             new_class = None
2022             self._reload_connection()
2023             if ctype not in supportedClassificationTypes:
2024                 raise vimconn.VimConnNotSupportedException(
2025                     'OpenStack VIM connector does not support provided Classification Type {}, supported ones are: '
2026                     '{}'.format(ctype, supportedClassificationTypes))
2027             if not self._validate_classification(ctype, definition):
2028                 raise vimconn.VimConnException(
2029                     'Incorrect Classification definition '
2030                     'for the type specified.')
2031             classification_dict = definition
2032             classification_dict['name'] = name
2033
2034             new_class = self.neutron.create_sfc_flow_classifier(
2035                 {'flow_classifier': classification_dict})
2036             return new_class['flow_classifier']['id']
2037         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2038                 neExceptions.NeutronException, ConnectionError) as e:
2039             self.logger.error(
2040                 'Creation of Classification failed.')
2041             self._format_exception(e)
2042
2043     def get_classification(self, class_id):
2044         self.logger.debug(" Getting Classification %s from VIM", class_id)
2045         filter_dict = {"id": class_id}
2046         class_list = self.get_classification_list(filter_dict)
2047         if len(class_list) == 0:
2048             raise vimconn.VimConnNotFoundException(
2049                 "Classification '{}' not found".format(class_id))
2050         elif len(class_list) > 1:
2051             raise vimconn.VimConnConflictException(
2052                 "Found more than one Classification with this criteria")
2053         classification = class_list[0]
2054         return classification
2055
2056     def get_classification_list(self, filter_dict={}):
2057         self.logger.debug("Getting Classifications from VIM filter: '%s'",
2058                           str(filter_dict))
2059         try:
2060             filter_dict_os = filter_dict.copy()
2061             self._reload_connection()
2062             if self.api_version3 and "tenant_id" in filter_dict_os:
2063                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
2064             classification_dict = self.neutron.list_sfc_flow_classifiers(
2065                 **filter_dict_os)
2066             classification_list = classification_dict["flow_classifiers"]
2067             self.__classification_os2mano(classification_list)
2068             return classification_list
2069         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2070                 neExceptions.NeutronException, ConnectionError) as e:
2071             self._format_exception(e)
2072
2073     def delete_classification(self, class_id):
2074         self.logger.debug("Deleting Classification '%s' from VIM", class_id)
2075         try:
2076             self._reload_connection()
2077             self.neutron.delete_sfc_flow_classifier(class_id)
2078             return class_id
2079         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
2080                 ksExceptions.ClientException, neExceptions.NeutronException,
2081                 ConnectionError) as e:
2082             self._format_exception(e)
2083
2084     def new_sfi(self, name, ingress_ports, egress_ports, sfc_encap=True):
2085         self.logger.debug("Adding a new Service Function Instance to VIM, named '%s'", name)
2086         try:
2087             new_sfi = None
2088             self._reload_connection()
2089             correlation = None
2090             if sfc_encap:
2091                 correlation = 'nsh'
2092             if len(ingress_ports) != 1:
2093                 raise vimconn.VimConnNotSupportedException(
2094                     "OpenStack VIM connector can only have "
2095                     "1 ingress port per SFI")
2096             if len(egress_ports) != 1:
2097                 raise vimconn.VimConnNotSupportedException(
2098                     "OpenStack VIM connector can only have "
2099                     "1 egress port per SFI")
2100             sfi_dict = {'name': name,
2101                         'ingress': ingress_ports[0],
2102                         'egress': egress_ports[0],
2103                         'service_function_parameters': {
2104                             'correlation': correlation}}
2105             new_sfi = self.neutron.create_sfc_port_pair({'port_pair': sfi_dict})
2106             return new_sfi['port_pair']['id']
2107         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2108                 neExceptions.NeutronException, ConnectionError) as e:
2109             if new_sfi:
2110                 try:
2111                     self.neutron.delete_sfc_port_pair(
2112                         new_sfi['port_pair']['id'])
2113                 except Exception:
2114                     self.logger.error(
2115                         'Creation of Service Function Instance failed, with '
2116                         'subsequent deletion failure as well.')
2117             self._format_exception(e)
2118
2119     def get_sfi(self, sfi_id):
2120         self.logger.debug('Getting Service Function Instance %s from VIM', sfi_id)
2121         filter_dict = {"id": sfi_id}
2122         sfi_list = self.get_sfi_list(filter_dict)
2123         if len(sfi_list) == 0:
2124             raise vimconn.VimConnNotFoundException("Service Function Instance '{}' not found".format(sfi_id))
2125         elif len(sfi_list) > 1:
2126             raise vimconn.VimConnConflictException(
2127                 'Found more than one Service Function Instance '
2128                 'with this criteria')
2129         sfi = sfi_list[0]
2130         return sfi
2131
2132     def get_sfi_list(self, filter_dict={}):
2133         self.logger.debug("Getting Service Function Instances from VIM filter: '%s'", str(filter_dict))
2134         try:
2135             self._reload_connection()
2136             filter_dict_os = filter_dict.copy()
2137             if self.api_version3 and "tenant_id" in filter_dict_os:
2138                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
2139             sfi_dict = self.neutron.list_sfc_port_pairs(**filter_dict_os)
2140             sfi_list = sfi_dict["port_pairs"]
2141             self.__sfi_os2mano(sfi_list)
2142             return sfi_list
2143         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2144                 neExceptions.NeutronException, ConnectionError) as e:
2145             self._format_exception(e)
2146
2147     def delete_sfi(self, sfi_id):
2148         self.logger.debug("Deleting Service Function Instance '%s' "
2149                           "from VIM", sfi_id)
2150         try:
2151             self._reload_connection()
2152             self.neutron.delete_sfc_port_pair(sfi_id)
2153             return sfi_id
2154         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
2155                 ksExceptions.ClientException, neExceptions.NeutronException,
2156                 ConnectionError) as e:
2157             self._format_exception(e)
2158
2159     def new_sf(self, name, sfis, sfc_encap=True):
2160         self.logger.debug("Adding a new Service Function to VIM, named '%s'", name)
2161         try:
2162             new_sf = None
2163             self._reload_connection()
2164             # correlation = None
2165             # if sfc_encap:
2166             #     correlation = 'nsh'
2167             for instance in sfis:
2168                 sfi = self.get_sfi(instance)
2169                 if sfi.get('sfc_encap') != sfc_encap:
2170                     raise vimconn.VimConnNotSupportedException(
2171                         "OpenStack VIM connector requires all SFIs of the "
2172                         "same SF to share the same SFC Encapsulation")
2173             sf_dict = {'name': name,
2174                        'port_pairs': sfis}
2175             new_sf = self.neutron.create_sfc_port_pair_group({
2176                 'port_pair_group': sf_dict})
2177             return new_sf['port_pair_group']['id']
2178         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2179                 neExceptions.NeutronException, ConnectionError) as e:
2180             if new_sf:
2181                 try:
2182                     self.neutron.delete_sfc_port_pair_group(
2183                         new_sf['port_pair_group']['id'])
2184                 except Exception:
2185                     self.logger.error(
2186                         'Creation of Service Function failed, with '
2187                         'subsequent deletion failure as well.')
2188             self._format_exception(e)
2189
2190     def get_sf(self, sf_id):
2191         self.logger.debug("Getting Service Function %s from VIM", sf_id)
2192         filter_dict = {"id": sf_id}
2193         sf_list = self.get_sf_list(filter_dict)
2194         if len(sf_list) == 0:
2195             raise vimconn.VimConnNotFoundException(
2196                 "Service Function '{}' not found".format(sf_id))
2197         elif len(sf_list) > 1:
2198             raise vimconn.VimConnConflictException(
2199                 "Found more than one Service Function with this criteria")
2200         sf = sf_list[0]
2201         return sf
2202
2203     def get_sf_list(self, filter_dict={}):
2204         self.logger.debug("Getting Service Function from VIM filter: '%s'",
2205                           str(filter_dict))
2206         try:
2207             self._reload_connection()
2208             filter_dict_os = filter_dict.copy()
2209             if self.api_version3 and "tenant_id" in filter_dict_os:
2210                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
2211             sf_dict = self.neutron.list_sfc_port_pair_groups(**filter_dict_os)
2212             sf_list = sf_dict["port_pair_groups"]
2213             self.__sf_os2mano(sf_list)
2214             return sf_list
2215         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2216                 neExceptions.NeutronException, ConnectionError) as e:
2217             self._format_exception(e)
2218
2219     def delete_sf(self, sf_id):
2220         self.logger.debug("Deleting Service Function '%s' from VIM", sf_id)
2221         try:
2222             self._reload_connection()
2223             self.neutron.delete_sfc_port_pair_group(sf_id)
2224             return sf_id
2225         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
2226                 ksExceptions.ClientException, neExceptions.NeutronException,
2227                 ConnectionError) as e:
2228             self._format_exception(e)
2229
2230     def new_sfp(self, name, classifications, sfs, sfc_encap=True, spi=None):
2231         self.logger.debug("Adding a new Service Function Path to VIM, named '%s'", name)
2232         try:
2233             new_sfp = None
2234             self._reload_connection()
2235             # In networking-sfc the MPLS encapsulation is legacy
2236             # should be used when no full SFC Encapsulation is intended
2237             correlation = 'mpls'
2238             if sfc_encap:
2239                 correlation = 'nsh'
2240             sfp_dict = {'name': name,
2241                         'flow_classifiers': classifications,
2242                         'port_pair_groups': sfs,
2243                         'chain_parameters': {'correlation': correlation}}
2244             if spi:
2245                 sfp_dict['chain_id'] = spi
2246             new_sfp = self.neutron.create_sfc_port_chain({'port_chain': sfp_dict})
2247             return new_sfp["port_chain"]["id"]
2248         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2249                 neExceptions.NeutronException, ConnectionError) as e:
2250             if new_sfp:
2251                 try:
2252                     self.neutron.delete_sfc_port_chain(new_sfp['port_chain']['id'])
2253                 except Exception:
2254                     self.logger.error(
2255                         'Creation of Service Function Path failed, with '
2256                         'subsequent deletion failure as well.')
2257             self._format_exception(e)
2258
2259     def get_sfp(self, sfp_id):
2260         self.logger.debug(" Getting Service Function Path %s from VIM", sfp_id)
2261         filter_dict = {"id": sfp_id}
2262         sfp_list = self.get_sfp_list(filter_dict)
2263         if len(sfp_list) == 0:
2264             raise vimconn.VimConnNotFoundException(
2265                 "Service Function Path '{}' not found".format(sfp_id))
2266         elif len(sfp_list) > 1:
2267             raise vimconn.VimConnConflictException(
2268                 "Found more than one Service Function Path with this criteria")
2269         sfp = sfp_list[0]
2270         return sfp
2271
2272     def get_sfp_list(self, filter_dict={}):
2273         self.logger.debug("Getting Service Function Paths from VIM filter: '%s'", str(filter_dict))
2274         try:
2275             self._reload_connection()
2276             filter_dict_os = filter_dict.copy()
2277             if self.api_version3 and "tenant_id" in filter_dict_os:
2278                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
2279             sfp_dict = self.neutron.list_sfc_port_chains(**filter_dict_os)
2280             sfp_list = sfp_dict["port_chains"]
2281             self.__sfp_os2mano(sfp_list)
2282             return sfp_list
2283         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2284                 neExceptions.NeutronException, ConnectionError) as e:
2285             self._format_exception(e)
2286
2287     def delete_sfp(self, sfp_id):
2288         self.logger.debug("Deleting Service Function Path '%s' from VIM", sfp_id)
2289         try:
2290             self._reload_connection()
2291             self.neutron.delete_sfc_port_chain(sfp_id)
2292             return sfp_id
2293         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
2294                 ksExceptions.ClientException, neExceptions.NeutronException,
2295                 ConnectionError) as e:
2296             self._format_exception(e)
2297
2298     def refresh_sfps_status(self, sfp_list):
2299         """Get the status of the service function path
2300            Params: the list of sfp identifiers
2301            Returns a dictionary with:
2302                 vm_id:          #VIM id of this service function path
2303                     status:     #Mandatory. Text with one of:
2304                                 #  DELETED (not found at vim)
2305                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
2306                                 #  OTHER (Vim reported other status not understood)
2307                                 #  ERROR (VIM indicates an ERROR status)
2308                                 #  ACTIVE,
2309                                 #  CREATING (on building process)
2310                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
2311                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)F
2312         """
2313         sfp_dict = {}
2314         self.logger.debug("refresh_sfps status: Getting tenant SFP information from VIM")
2315         for sfp_id in sfp_list:
2316             sfp = {}
2317             try:
2318                 sfp_vim = self.get_sfp(sfp_id)
2319                 if sfp_vim['spi']:
2320                     sfp['status'] = vmStatus2manoFormat['ACTIVE']
2321                 else:
2322                     sfp['status'] = "OTHER"
2323                     sfp['error_msg'] = "VIM status reported " + sfp['status']
2324
2325                 sfp['vim_info'] = self.serialize(sfp_vim)
2326
2327                 if sfp_vim.get('fault'):
2328                     sfp['error_msg'] = str(sfp_vim['fault'])
2329
2330             except vimconn.VimConnNotFoundException as e:
2331                 self.logger.error("Exception getting sfp status: %s", str(e))
2332                 sfp['status'] = "DELETED"
2333                 sfp['error_msg'] = str(e)
2334             except vimconn.VimConnException as e:
2335                 self.logger.error("Exception getting sfp status: %s", str(e))
2336                 sfp['status'] = "VIM_ERROR"
2337                 sfp['error_msg'] = str(e)
2338             sfp_dict[sfp_id] = sfp
2339         return sfp_dict
2340
2341     def refresh_sfis_status(self, sfi_list):
2342         """Get the status of the service function instances
2343            Params: the list of sfi identifiers
2344            Returns a dictionary with:
2345                 vm_id:          #VIM id of this service function instance
2346                     status:     #Mandatory. Text with one of:
2347                                 #  DELETED (not found at vim)
2348                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
2349                                 #  OTHER (Vim reported other status not understood)
2350                                 #  ERROR (VIM indicates an ERROR status)
2351                                 #  ACTIVE,
2352                                 #  CREATING (on building process)
2353                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
2354                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
2355         """
2356         sfi_dict = {}
2357         self.logger.debug("refresh_sfis status: Getting tenant sfi information from VIM")
2358         for sfi_id in sfi_list:
2359             sfi = {}
2360             try:
2361                 sfi_vim = self.get_sfi(sfi_id)
2362                 if sfi_vim:
2363                     sfi['status'] = vmStatus2manoFormat['ACTIVE']
2364                 else:
2365                     sfi['status'] = "OTHER"
2366                     sfi['error_msg'] = "VIM status reported " + sfi['status']
2367
2368                 sfi['vim_info'] = self.serialize(sfi_vim)
2369
2370                 if sfi_vim.get('fault'):
2371                     sfi['error_msg'] = str(sfi_vim['fault'])
2372
2373             except vimconn.VimConnNotFoundException as e:
2374                 self.logger.error("Exception getting sfi status: %s", str(e))
2375                 sfi['status'] = "DELETED"
2376                 sfi['error_msg'] = str(e)
2377             except vimconn.VimConnException as e:
2378                 self.logger.error("Exception getting sfi status: %s", str(e))
2379                 sfi['status'] = "VIM_ERROR"
2380                 sfi['error_msg'] = str(e)
2381             sfi_dict[sfi_id] = sfi
2382         return sfi_dict
2383
2384     def refresh_sfs_status(self, sf_list):
2385         """Get the status of the service functions
2386            Params: the list of sf identifiers
2387            Returns a dictionary with:
2388                 vm_id:          #VIM id of this service function
2389                     status:     #Mandatory. Text with one of:
2390                                 #  DELETED (not found at vim)
2391                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
2392                                 #  OTHER (Vim reported other status not understood)
2393                                 #  ERROR (VIM indicates an ERROR status)
2394                                 #  ACTIVE,
2395                                 #  CREATING (on building process)
2396                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
2397                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
2398         """
2399         sf_dict = {}
2400         self.logger.debug("refresh_sfs status: Getting tenant sf information from VIM")
2401         for sf_id in sf_list:
2402             sf = {}
2403             try:
2404                 sf_vim = self.get_sf(sf_id)
2405                 if sf_vim:
2406                     sf['status'] = vmStatus2manoFormat['ACTIVE']
2407                 else:
2408                     sf['status'] = "OTHER"
2409                     sf['error_msg'] = "VIM status reported " + sf_vim['status']
2410
2411                 sf['vim_info'] = self.serialize(sf_vim)
2412
2413                 if sf_vim.get('fault'):
2414                     sf['error_msg'] = str(sf_vim['fault'])
2415
2416             except vimconn.VimConnNotFoundException as e:
2417                 self.logger.error("Exception getting sf status: %s", str(e))
2418                 sf['status'] = "DELETED"
2419                 sf['error_msg'] = str(e)
2420             except vimconn.VimConnException as e:
2421                 self.logger.error("Exception getting sf status: %s", str(e))
2422                 sf['status'] = "VIM_ERROR"
2423                 sf['error_msg'] = str(e)
2424             sf_dict[sf_id] = sf
2425         return sf_dict
2426
2427     def refresh_classifications_status(self, classification_list):
2428         """Get the status of the classifications
2429            Params: the list of classification identifiers
2430            Returns a dictionary with:
2431                 vm_id:          #VIM id of this classifier
2432                     status:     #Mandatory. Text with one of:
2433                                 #  DELETED (not found at vim)
2434                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
2435                                 #  OTHER (Vim reported other status not understood)
2436                                 #  ERROR (VIM indicates an ERROR status)
2437                                 #  ACTIVE,
2438                                 #  CREATING (on building process)
2439                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
2440                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
2441         """
2442         classification_dict = {}
2443         self.logger.debug("refresh_classifications status: Getting tenant classification information from VIM")
2444         for classification_id in classification_list:
2445             classification = {}
2446             try:
2447                 classification_vim = self.get_classification(classification_id)
2448                 if classification_vim:
2449                     classification['status'] = vmStatus2manoFormat['ACTIVE']
2450                 else:
2451                     classification['status'] = "OTHER"
2452                     classification['error_msg'] = "VIM status reported " + classification['status']
2453
2454                 classification['vim_info'] = self.serialize(classification_vim)
2455
2456                 if classification_vim.get('fault'):
2457                     classification['error_msg'] = str(classification_vim['fault'])
2458
2459             except vimconn.VimConnNotFoundException as e:
2460                 self.logger.error("Exception getting classification status: %s", str(e))
2461                 classification['status'] = "DELETED"
2462                 classification['error_msg'] = str(e)
2463             except vimconn.VimConnException as e:
2464                 self.logger.error("Exception getting classification status: %s", str(e))
2465                 classification['status'] = "VIM_ERROR"
2466                 classification['error_msg'] = str(e)
2467             classification_dict[classification_id] = classification
2468         return classification_dict