RO-VIM-openstack/osm_rovim_openstack/vimconn_openstack.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 ##
  20
  21 """
  22 osconnector implements all the methods to interact with openstack using the python-neutronclient.
  23
  24 For the VNF forwarding graph, The OpenStack VIM connector calls the
  25 networking-sfc Neutron extension methods, whose resources are mapped
  26 to the VIM connector's SFC resources as follows:
  27 - Classification (OSM) -> Flow Classifier (Neutron)
  28 - Service Function Instance (OSM) -> Port Pair (Neutron)
  29 - Service Function (OSM) -> Port Pair Group (Neutron)
  30 - Service Function Path (OSM) -> Port Chain (Neutron)
  31 """
  32
  33 from osm_ro_plugin import vimconn
  34 # import json
  35 import logging
  36 import netaddr
  37 import time
  38 import yaml
  39 import random
  40 import re
  41 import copy
  42 from pprint import pformat
  43
  44 from novaclient import client as nClient, exceptions as nvExceptions
  45 from keystoneauth1.identity import v2, v3
  46 from keystoneauth1 import session
  47 import keystoneclient.exceptions as ksExceptions
  48 import keystoneclient.v3.client as ksClient_v3
  49 import keystoneclient.v2_0.client as ksClient_v2
  50 from glanceclient import client as glClient
  51 import glanceclient.exc as gl1Exceptions
  52 from cinderclient import client as cClient
  53 from http.client import HTTPException   # TODO py3 check that this base exception matches python2 httplib.HTTPException
  54 from neutronclient.neutron import client as neClient
  55 from neutronclient.common import exceptions as neExceptions
  56 from requests.exceptions import ConnectionError
  57
  58 __author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes, xFlow Research, Igor D.C., Eduardo Sousa"
  59 __date__ = "$22-sep-2017 23:59:59$"
  60
  61 """contain the openstack virtual machine status to openmano status"""
  62 vmStatus2manoFormat = {'ACTIVE': 'ACTIVE',
  63                        'PAUSED': 'PAUSED',
  64                        'SUSPENDED': 'SUSPENDED',
  65                        'SHUTOFF': 'INACTIVE',
  66                        'BUILD': 'BUILD',
  67                        'ERROR': 'ERROR',
  68                        'DELETED': 'DELETED'
  69                        }
  70 netStatus2manoFormat = {'ACTIVE': 'ACTIVE',
  71                         'PAUSED': 'PAUSED',
  72                         'INACTIVE': 'INACTIVE',
  73                         'BUILD': 'BUILD',
  74                         'ERROR': 'ERROR',
  75                         'DELETED': 'DELETED'
  76                         }
  77
  78 supportedClassificationTypes = ['legacy_flow_classifier']
  79
  80 # global var to have a timeout creating and deleting volumes
  81 volume_timeout = 1800
  82 server_timeout = 1800
  83
  84
  85 class SafeDumper(yaml.SafeDumper):
  86     def represent_data(self, data):
  87         # Openstack APIs use custom subclasses of dict and YAML safe dumper
  88         # is designed to not handle that (reference issue 142 of pyyaml)
  89         if isinstance(data, dict) and data.__class__ != dict:
  90             # A simple solution is to convert those items back to dicts
  91             data = dict(data.items())
  92
  93         return super(SafeDumper, self).represent_data(data)
  94
  95
  96 class vimconnector(vimconn.VimConnector):
  97     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None,
  98                  log_level=None, config={}, persistent_info={}):
  99         """using common constructor parameters. In this case
 100         'url' is the keystone authorization url,
 101         'url_admin' is not use
 102         """
 103         api_version = config.get('APIversion')
 104         if api_version and api_version not in ('v3.3', 'v2.0', '2', '3'):
 105             raise vimconn.VimConnException("Invalid value '{}' for config:APIversion. "
 106                                            "Allowed values are 'v3.3', 'v2.0', '2' or '3'".format(api_version))
 107         vim_type = config.get('vim_type')
 108         if vim_type and vim_type not in ('vio', 'VIO'):
 109             raise vimconn.VimConnException("Invalid value '{}' for config:vim_type."
 110                                            "Allowed values are 'vio' or 'VIO'".format(vim_type))
 111
 112         if config.get('dataplane_net_vlan_range') is not None:
 113             # validate vlan ranges provided by user
 114             self._validate_vlan_ranges(config.get('dataplane_net_vlan_range'), 'dataplane_net_vlan_range')
 115
 116         if config.get('multisegment_vlan_range') is not None:
 117             # validate vlan ranges provided by user
 118             self._validate_vlan_ranges(config.get('multisegment_vlan_range'), 'multisegment_vlan_range')
 119
 120         vimconn.VimConnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level,
 121                                       config)
 122
 123         if self.config.get("insecure") and self.config.get("ca_cert"):
 124             raise vimconn.VimConnException("options insecure and ca_cert are mutually exclusive")
 125         self.verify = True
 126         if self.config.get("insecure"):
 127             self.verify = False
 128         if self.config.get("ca_cert"):
 129             self.verify = self.config.get("ca_cert")
 130
 131         if not url:
 132             raise TypeError('url param can not be NoneType')
 133         self.persistent_info = persistent_info
 134         self.availability_zone = persistent_info.get('availability_zone', None)
 135         self.session = persistent_info.get('session', {'reload_client': True})
 136         self.my_tenant_id = self.session.get('my_tenant_id')
 137         self.nova = self.session.get('nova')
 138         self.neutron = self.session.get('neutron')
 139         self.cinder = self.session.get('cinder')
 140         self.glance = self.session.get('glance')
 141         # self.glancev1 = self.session.get('glancev1')
 142         self.keystone = self.session.get('keystone')
 143         self.api_version3 = self.session.get('api_version3')
 144         self.vim_type = self.config.get("vim_type")
 145         if self.vim_type:
 146             self.vim_type = self.vim_type.upper()
 147         if self.config.get("use_internal_endpoint"):
 148             self.endpoint_type = "internalURL"
 149         else:
 150             self.endpoint_type = None
 151
 152         logging.getLogger('urllib3').setLevel(logging.WARNING)
 153         logging.getLogger('keystoneauth').setLevel(logging.WARNING)
 154         logging.getLogger('novaclient').setLevel(logging.WARNING)
 155         self.logger = logging.getLogger('openmano.vim.openstack')
 156
 157         # allow security_groups to be a list or a single string
 158         if isinstance(self.config.get('security_groups'), str):
 159             self.config['security_groups'] = [self.config['security_groups']]
 160         self.security_groups_id = None
 161
 162         # ###### VIO Specific Changes #########
 163         if self.vim_type == "VIO":
 164             self.logger = logging.getLogger('openmano.vim.vio')
 165
 166         if log_level:
 167             self.logger.setLevel(getattr(logging, log_level))
 168
 169     def __getitem__(self, index):
 170         """Get individuals parameters.
 171         Throw KeyError"""
 172         if index == 'project_domain_id':
 173             return self.config.get("project_domain_id")
 174         elif index == 'user_domain_id':
 175             return self.config.get("user_domain_id")
 176         else:
 177             return vimconn.VimConnector.__getitem__(self, index)
 178
 179     def __setitem__(self, index, value):
 180         """Set individuals parameters and it is marked as dirty so to force connection reload.
 181         Throw KeyError"""
 182         if index == 'project_domain_id':
 183             self.config["project_domain_id"] = value
 184         elif index == 'user_domain_id':
 185             self.config["user_domain_id"] = value
 186         else:
 187             vimconn.VimConnector.__setitem__(self, index, value)
 188         self.session['reload_client'] = True
 189
 190     def serialize(self, value):
 191         """Serialization of python basic types.
 192
 193         In the case value is not serializable a message will be logged and a
 194         simple representation of the data that cannot be converted back to
 195         python is returned.
 196         """
 197         if isinstance(value, str):
 198             return value
 199
 200         try:
 201             return yaml.dump(value, Dumper=SafeDumper,
 202                              default_flow_style=True, width=256)
 203         except yaml.representer.RepresenterError:
 204             self.logger.debug('The following entity cannot be serialized in YAML:\n\n%s\n\n', pformat(value),
 205                               exc_info=True)
 206             return str(value)
 207
 208     def _reload_connection(self):
 209         """Called before any operation, it check if credentials has changed
 210         Throw keystoneclient.apiclient.exceptions.AuthorizationFailure
 211         """
 212         # TODO control the timing and possible token timeout, but it seams that python client does this task for us :-)
 213         if self.session['reload_client']:
 214             if self.config.get('APIversion'):
 215                 self.api_version3 = self.config['APIversion'] == 'v3.3' or self.config['APIversion'] == '3'
 216             else:  # get from ending auth_url that end with v3 or with v2.0
 217                 self.api_version3 = self.url.endswith("/v3") or self.url.endswith("/v3/")
 218             self.session['api_version3'] = self.api_version3
 219             if self.api_version3:
 220                 if self.config.get('project_domain_id') or self.config.get('project_domain_name'):
 221                     project_domain_id_default = None
 222                 else:
 223                     project_domain_id_default = 'default'
 224                 if self.config.get('user_domain_id') or self.config.get('user_domain_name'):
 225                     user_domain_id_default = None
 226                 else:
 227                     user_domain_id_default = 'default'
 228                 auth = v3.Password(auth_url=self.url,
 229                                    username=self.user,
 230                                    password=self.passwd,
 231                                    project_name=self.tenant_name,
 232                                    project_id=self.tenant_id,
 233                                    project_domain_id=self.config.get('project_domain_id', project_domain_id_default),
 234                                    user_domain_id=self.config.get('user_domain_id', user_domain_id_default),
 235                                    project_domain_name=self.config.get('project_domain_name'),
 236                                    user_domain_name=self.config.get('user_domain_name'))
 237             else:
 238                 auth = v2.Password(auth_url=self.url,
 239                                    username=self.user,
 240                                    password=self.passwd,
 241                                    tenant_name=self.tenant_name,
 242                                    tenant_id=self.tenant_id)
 243             sess = session.Session(auth=auth, verify=self.verify)
 244             # addedd region_name to keystone, nova, neutron and cinder to support distributed cloud for Wind River
 245             # Titanium cloud and StarlingX
 246             region_name = self.config.get('region_name')
 247             if self.api_version3:
 248                 self.keystone = ksClient_v3.Client(session=sess, endpoint_type=self.endpoint_type,
 249                                                    region_name=region_name)
 250             else:
 251                 self.keystone = ksClient_v2.Client(session=sess, endpoint_type=self.endpoint_type)
 252             self.session['keystone'] = self.keystone
 253             # In order to enable microversion functionality an explicit microversion must be specified in 'config'.
 254             # This implementation approach is due to the warning message in
 255             # https://developer.openstack.org/api-guide/compute/microversions.html
 256             # where it is stated that microversion backwards compatibility is not guaranteed and clients should
 257             # always require an specific microversion.
 258             # To be able to use 'device role tagging' functionality define 'microversion: 2.32' in datacenter config
 259             version = self.config.get("microversion")
 260             if not version:
 261                 version = "2.1"
 262             # addedd region_name to keystone, nova, neutron and cinder to support distributed cloud for Wind River
 263             # Titanium cloud and StarlingX
 264             self.nova = self.session['nova'] = nClient.Client(str(version), session=sess,
 265                                                               endpoint_type=self.endpoint_type, region_name=region_name)
 266             self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess,
 267                                                                      endpoint_type=self.endpoint_type,
 268                                                                      region_name=region_name)
 269             self.cinder = self.session['cinder'] = cClient.Client(2, session=sess, endpoint_type=self.endpoint_type,
 270                                                                   region_name=region_name)
 271             try:
 272                 self.my_tenant_id = self.session['my_tenant_id'] = sess.get_project_id()
 273             except Exception:
 274                 self.logger.error("Cannot get project_id from session", exc_info=True)
 275             if self.endpoint_type == "internalURL":
 276                 glance_service_id = self.keystone.services.list(name="glance")[0].id
 277                 glance_endpoint = self.keystone.endpoints.list(glance_service_id, interface="internal")[0].url
 278             else:
 279                 glance_endpoint = None
 280             self.glance = self.session['glance'] = glClient.Client(2, session=sess, endpoint=glance_endpoint)
 281             # using version 1 of glance client in new_image()
 282             # self.glancev1 = self.session['glancev1'] = glClient.Client('1', session=sess,
 283             #                                                            endpoint=glance_endpoint)
 284             self.session['reload_client'] = False
 285             self.persistent_info['session'] = self.session
 286             # add availablity zone info inside  self.persistent_info
 287             self._set_availablity_zones()
 288             self.persistent_info['availability_zone'] = self.availability_zone
 289             self.security_groups_id = None  # force to get again security_groups_ids next time they are needed
 290
 291     def __net_os2mano(self, net_list_dict):
 292         """Transform the net openstack format to mano format
 293         net_list_dict can be a list of dict or a single dict"""
 294         if type(net_list_dict) is dict:
 295             net_list_ = (net_list_dict,)
 296         elif type(net_list_dict) is list:
 297             net_list_ = net_list_dict
 298         else:
 299             raise TypeError("param net_list_dict must be a list or a dictionary")
 300         for net in net_list_:
 301             if net.get('provider:network_type') == "vlan":
 302                 net['type'] = 'data'
 303             else:
 304                 net['type'] = 'bridge'
 305
 306     def __classification_os2mano(self, class_list_dict):
 307         """Transform the openstack format (Flow Classifier) to mano format
 308         (Classification) class_list_dict can be a list of dict or a single dict
 309         """
 310         if isinstance(class_list_dict, dict):
 311             class_list_ = [class_list_dict]
 312         elif isinstance(class_list_dict, list):
 313             class_list_ = class_list_dict
 314         else:
 315             raise TypeError("param class_list_dict must be a list or a dictionary")
 316         for classification in class_list_:
 317             id = classification.pop('id')
 318             name = classification.pop('name')
 319             description = classification.pop('description')
 320             project_id = classification.pop('project_id')
 321             tenant_id = classification.pop('tenant_id')
 322             original_classification = copy.deepcopy(classification)
 323             classification.clear()
 324             classification['ctype'] = 'legacy_flow_classifier'
 325             classification['definition'] = original_classification
 326             classification['id'] = id
 327             classification['name'] = name
 328             classification['description'] = description
 329             classification['project_id'] = project_id
 330             classification['tenant_id'] = tenant_id
 331
 332     def __sfi_os2mano(self, sfi_list_dict):
 333         """Transform the openstack format (Port Pair) to mano format (SFI)
 334         sfi_list_dict can be a list of dict or a single dict
 335         """
 336         if isinstance(sfi_list_dict, dict):
 337             sfi_list_ = [sfi_list_dict]
 338         elif isinstance(sfi_list_dict, list):
 339             sfi_list_ = sfi_list_dict
 340         else:
 341             raise TypeError(
 342                 "param sfi_list_dict must be a list or a dictionary")
 343         for sfi in sfi_list_:
 344             sfi['ingress_ports'] = []
 345             sfi['egress_ports'] = []
 346             if sfi.get('ingress'):
 347                 sfi['ingress_ports'].append(sfi['ingress'])
 348             if sfi.get('egress'):
 349                 sfi['egress_ports'].append(sfi['egress'])
 350             del sfi['ingress']
 351             del sfi['egress']
 352             params = sfi.get('service_function_parameters')
 353             sfc_encap = False
 354             if params:
 355                 correlation = params.get('correlation')
 356                 if correlation:
 357                     sfc_encap = True
 358             sfi['sfc_encap'] = sfc_encap
 359             del sfi['service_function_parameters']
 360
 361     def __sf_os2mano(self, sf_list_dict):
 362         """Transform the openstack format (Port Pair Group) to mano format (SF)
 363         sf_list_dict can be a list of dict or a single dict
 364         """
 365         if isinstance(sf_list_dict, dict):
 366             sf_list_ = [sf_list_dict]
 367         elif isinstance(sf_list_dict, list):
 368             sf_list_ = sf_list_dict
 369         else:
 370             raise TypeError(
 371                 "param sf_list_dict must be a list or a dictionary")
 372         for sf in sf_list_:
 373             del sf['port_pair_group_parameters']
 374             sf['sfis'] = sf['port_pairs']
 375             del sf['port_pairs']
 376
 377     def __sfp_os2mano(self, sfp_list_dict):
 378         """Transform the openstack format (Port Chain) to mano format (SFP)
 379         sfp_list_dict can be a list of dict or a single dict
 380         """
 381         if isinstance(sfp_list_dict, dict):
 382             sfp_list_ = [sfp_list_dict]
 383         elif isinstance(sfp_list_dict, list):
 384             sfp_list_ = sfp_list_dict
 385         else:
 386             raise TypeError(
 387                 "param sfp_list_dict must be a list or a dictionary")
 388         for sfp in sfp_list_:
 389             params = sfp.pop('chain_parameters')
 390             sfc_encap = False
 391             if params:
 392                 correlation = params.get('correlation')
 393                 if correlation:
 394                     sfc_encap = True
 395             sfp['sfc_encap'] = sfc_encap
 396             sfp['spi'] = sfp.pop('chain_id')
 397             sfp['classifications'] = sfp.pop('flow_classifiers')
 398             sfp['service_functions'] = sfp.pop('port_pair_groups')
 399
 400     # placeholder for now; read TODO note below
 401     def _validate_classification(self, type, definition):
 402         # only legacy_flow_classifier Type is supported at this point
 403         return True
 404         # TODO(igordcard): this method should be an abstract method of an
 405         # abstract Classification class to be implemented by the specific
 406         # Types. Also, abstract vimconnector should call the validation
 407         # method before the implemented VIM connectors are called.
 408
 409     def _format_exception(self, exception):
 410         """Transform a keystone, nova, neutron  exception into a vimconn exception discovering the cause"""
 411
 412         message_error = str(exception)
 413         tip = ""
 414
 415         if isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound, ksExceptions.NotFound,
 416                                   gl1Exceptions.HTTPNotFound)):
 417             raise vimconn.VimConnNotFoundException(type(exception).__name__ + ": " + message_error)
 418         elif isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError,
 419                                     ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed)):
 420             if type(exception).__name__ == "SSLError":
 421                 tip = " (maybe option 'insecure' must be added to the VIM)"
 422             raise vimconn.VimConnConnectionException("Invalid URL or credentials{}: {}".format(tip, message_error))
 423         elif isinstance(exception, (KeyError, nvExceptions.BadRequest, ksExceptions.BadRequest)):
 424             raise vimconn.VimConnException(type(exception).__name__ + ": " + message_error)
 425         elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException,
 426                                     neExceptions.NeutronException)):
 427             raise vimconn.VimConnUnexpectedResponse(type(exception).__name__ + ": " + message_error)
 428         elif isinstance(exception, nvExceptions.Conflict):
 429             raise vimconn.VimConnConflictException(type(exception).__name__ + ": " + message_error)
 430         elif isinstance(exception, vimconn.VimConnException):
 431             raise exception
 432         else:  # ()
 433             self.logger.error("General Exception " + message_error, exc_info=True)
 434             raise vimconn.VimConnConnectionException(type(exception).__name__ + ": " + message_error)
 435
 436     def _get_ids_from_name(self):
 437         """
 438          Obtain ids from name of tenant and security_groups. Store at self .security_groups_id"
 439         :return: None
 440         """
 441         # get tenant_id if only tenant_name is supplied
 442         self._reload_connection()
 443         if not self.my_tenant_id:
 444             raise vimconn.VimConnConnectionException("Error getting tenant information from name={} id={}".
 445                                                      format(self.tenant_name, self.tenant_id))
 446         if self.config.get('security_groups') and not self.security_groups_id:
 447             # convert from name to id
 448             neutron_sg_list = self.neutron.list_security_groups(tenant_id=self.my_tenant_id)["security_groups"]
 449
 450             self.security_groups_id = []
 451             for sg in self.config.get('security_groups'):
 452                 for neutron_sg in neutron_sg_list:
 453                     if sg in (neutron_sg["id"], neutron_sg["name"]):
 454                         self.security_groups_id.append(neutron_sg["id"])
 455                         break
 456                 else:
 457                     self.security_groups_id = None
 458                     raise vimconn.VimConnConnectionException("Not found security group {} for this tenant".format(sg))
 459
 460     def check_vim_connectivity(self):
 461         # just get network list to check connectivity and credentials
 462         self.get_network_list(filter_dict={})
 463
 464     def get_tenant_list(self, filter_dict={}):
 465         """Obtain tenants of VIM
 466         filter_dict can contain the following keys:
 467             name: filter by tenant name
 468             id: filter by tenant uuid/id
 469             <other VIM specific>
 470         Returns the tenant list of dictionaries: [{'name':'<name>, 'id':'<id>, ...}, ...]
 471         """
 472         self.logger.debug("Getting tenants from VIM filter: '%s'", str(filter_dict))
 473         try:
 474             self._reload_connection()
 475             if self.api_version3:
 476                 project_class_list = self.keystone.projects.list(name=filter_dict.get("name"))
 477             else:
 478                 project_class_list = self.keystone.tenants.findall(**filter_dict)
 479             project_list = []
 480             for project in project_class_list:
 481                 if filter_dict.get('id') and filter_dict["id"] != project.id:
 482                     continue
 483                 project_list.append(project.to_dict())
 484             return project_list
 485         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError) as e:
 486             self._format_exception(e)
 487
 488     def new_tenant(self, tenant_name, tenant_description):
 489         """Adds a new tenant to openstack VIM. Returns the tenant identifier"""
 490         self.logger.debug("Adding a new tenant name: %s", tenant_name)
 491         try:
 492             self._reload_connection()
 493             if self.api_version3:
 494                 project = self.keystone.projects.create(tenant_name, self.config.get("project_domain_id", "default"),
 495                                                         description=tenant_description, is_domain=False)
 496             else:
 497                 project = self.keystone.tenants.create(tenant_name, tenant_description)
 498             return project.id
 499         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ksExceptions.BadRequest, ConnectionError)\
 500                 as e:
 501             self._format_exception(e)
 502
 503     def delete_tenant(self, tenant_id):
 504         """Delete a tenant from openstack VIM. Returns the old tenant identifier"""
 505         self.logger.debug("Deleting tenant %s from VIM", tenant_id)
 506         try:
 507             self._reload_connection()
 508             if self.api_version3:
 509                 self.keystone.projects.delete(tenant_id)
 510             else:
 511                 self.keystone.tenants.delete(tenant_id)
 512             return tenant_id
 513         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ksExceptions.NotFound, ConnectionError)\
 514                 as e:
 515             self._format_exception(e)
 516
 517     def new_network(self, net_name, net_type, ip_profile=None, shared=False, provider_network_profile=None):
 518         """Adds a tenant network to VIM
 519         Params:
 520             'net_name': name of the network
 521             'net_type': one of:
 522                 'bridge': overlay isolated network
 523                 'data':   underlay E-LAN network for Passthrough and SRIOV interfaces
 524                 'ptp':    underlay E-LINE network for Passthrough and SRIOV interfaces.
 525             'ip_profile': is a dict containing the IP parameters of the network
 526                 'ip_version': can be "IPv4" or "IPv6" (Currently only IPv4 is implemented)
 527                 'subnet_address': ip_prefix_schema, that is X.X.X.X/Y
 528                 'gateway_address': (Optional) ip_schema, that is X.X.X.X
 529                 'dns_address': (Optional) comma separated list of ip_schema, e.g. X.X.X.X[,X,X,X,X]
 530                 'dhcp_enabled': True or False
 531                 'dhcp_start_address': ip_schema, first IP to grant
 532                 'dhcp_count': number of IPs to grant.
 533             'shared': if this network can be seen/use by other tenants/organization
 534             'provider_network_profile': (optional) contains {segmentation-id: vlan, network-type: vlan|vxlan,
 535                                                              physical-network: physnet-label}
 536         Returns a tuple with the network identifier and created_items, or raises an exception on error
 537             created_items can be None or a dictionary where this method can include key-values that will be passed to
 538             the method delete_network. Can be used to store created segments, created l2gw connections, etc.
 539             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
 540             as not present.
 541         """
 542         self.logger.debug("Adding a new network to VIM name '%s', type '%s'", net_name, net_type)
 543         # self.logger.debug(">>>>>>>>>>>>>>>>>> IP profile %s", str(ip_profile))
 544
 545         try:
 546             vlan = None
 547             if provider_network_profile:
 548                 vlan = provider_network_profile.get("segmentation-id")
 549             new_net = None
 550             created_items = {}
 551             self._reload_connection()
 552             network_dict = {'name': net_name, 'admin_state_up': True}
 553             if net_type in ("data", "ptp"):
 554                 provider_physical_network = None
 555                 if provider_network_profile and provider_network_profile.get("physical-network"):
 556                     provider_physical_network = provider_network_profile.get("physical-network")
 557                     # provider-network must be one of the dataplane_physcial_netowrk if this is a list. If it is string
 558                     # or not declared, just ignore the checking
 559                     if isinstance(self.config.get('dataplane_physical_net'), (tuple, list)) and \
 560                             provider_physical_network not in self.config['dataplane_physical_net']:
 561                         raise vimconn.VimConnConflictException(
 562                             "Invalid parameter 'provider-network:physical-network' for network creation. '{}' is not "
 563                             "one of the declared list at VIM_config:dataplane_physical_net".format(
 564                                 provider_physical_network))
 565                 if not provider_physical_network:  # use the default dataplane_physical_net
 566                     provider_physical_network = self.config.get('dataplane_physical_net')
 567                     # if it is non empty list, use the first value. If it is a string use the value directly
 568                     if isinstance(provider_physical_network, (tuple, list)) and provider_physical_network:
 569                         provider_physical_network = provider_physical_network[0]
 570
 571                 if not provider_physical_network:
 572                     raise vimconn.VimConnConflictException(
 573                         "missing information needed for underlay networks. Provide 'dataplane_physical_net' "
 574                         "configuration at VIM or use the NS instantiation parameter 'provider-network.physical-network'"
 575                         " for the VLD")
 576
 577                 if not self.config.get('multisegment_support'):
 578                     network_dict["provider:physical_network"] = provider_physical_network
 579                     if provider_network_profile and "network-type" in provider_network_profile:
 580                         network_dict["provider:network_type"] = provider_network_profile["network-type"]
 581                     else:
 582                         network_dict["provider:network_type"] = self.config.get('dataplane_network_type', 'vlan')
 583                     if vlan:
 584                         network_dict["provider:segmentation_id"] = vlan
 585                 else:
 586                     # Multi-segment case
 587                     segment_list = []
 588                     segment1_dict = {
 589                         "provider:physical_network": '',
 590                         "provider:network_type": 'vxlan'
 591                     }
 592                     segment_list.append(segment1_dict)
 593                     segment2_dict = {
 594                         "provider:physical_network": provider_physical_network,
 595                         "provider:network_type": "vlan"
 596                     }
 597                     if vlan:
 598                         segment2_dict["provider:segmentation_id"] = vlan
 599                     elif self.config.get('multisegment_vlan_range'):
 600                         vlanID = self._generate_multisegment_vlanID()
 601                         segment2_dict["provider:segmentation_id"] = vlanID
 602                     # else
 603                     #     raise vimconn.VimConnConflictException(
 604                     #         "You must provide 'multisegment_vlan_range' at config dict before creating a multisegment
 605                     #         network")
 606                     segment_list.append(segment2_dict)
 607                     network_dict["segments"] = segment_list
 608
 609                 # VIO Specific Changes. It needs a concrete VLAN
 610                 if self.vim_type == "VIO" and vlan is None:
 611                     if self.config.get('dataplane_net_vlan_range') is None:
 612                         raise vimconn.VimConnConflictException(
 613                             "You must provide 'dataplane_net_vlan_range' in format [start_ID - end_ID] at VIM_config "
 614                             "for creating underlay networks")
 615                     network_dict["provider:segmentation_id"] = self._generate_vlanID()
 616
 617             network_dict["shared"] = shared
 618             if self.config.get("disable_network_port_security"):
 619                 network_dict["port_security_enabled"] = False
 620             new_net = self.neutron.create_network({'network': network_dict})
 621             # print new_net
 622             # create subnetwork, even if there is no profile
 623             if not ip_profile:
 624                 ip_profile = {}
 625             if not ip_profile.get('subnet_address'):
 626                 # Fake subnet is required
 627                 subnet_rand = random.randint(0, 255)
 628                 ip_profile['subnet_address'] = "192.168.{}.0/24".format(subnet_rand)
 629             if 'ip_version' not in ip_profile:
 630                 ip_profile['ip_version'] = "IPv4"
 631             subnet = {"name": net_name+"-subnet",
 632                       "network_id": new_net["network"]["id"],
 633                       "ip_version": 4 if ip_profile['ip_version'] == "IPv4" else 6,
 634                       "cidr": ip_profile['subnet_address']
 635                       }
 636             # Gateway should be set to None if not needed. Otherwise openstack assigns one by default
 637             if ip_profile.get('gateway_address'):
 638                 subnet['gateway_ip'] = ip_profile['gateway_address']
 639             else:
 640                 subnet['gateway_ip'] = None
 641             if ip_profile.get('dns_address'):
 642                 subnet['dns_nameservers'] = ip_profile['dns_address'].split(";")
 643             if 'dhcp_enabled' in ip_profile:
 644                 subnet['enable_dhcp'] = False if \
 645                     ip_profile['dhcp_enabled'] == "false" or ip_profile['dhcp_enabled'] is False else True
 646             if ip_profile.get('dhcp_start_address'):
 647                 subnet['allocation_pools'] = []
 648                 subnet['allocation_pools'].append(dict())
 649                 subnet['allocation_pools'][0]['start'] = ip_profile['dhcp_start_address']
 650             if ip_profile.get('dhcp_count'):
 651                 # parts = ip_profile['dhcp_start_address'].split('.')
 652                 # ip_int = (int(parts[0]) << 24) + (int(parts[1]) << 16) + (int(parts[2]) << 8) + int(parts[3])
 653                 ip_int = int(netaddr.IPAddress(ip_profile['dhcp_start_address']))
 654                 ip_int += ip_profile['dhcp_count'] - 1
 655                 ip_str = str(netaddr.IPAddress(ip_int))
 656                 subnet['allocation_pools'][0]['end'] = ip_str
 657             # self.logger.debug(">>>>>>>>>>>>>>>>>> Subnet: %s", str(subnet))
 658             self.neutron.create_subnet({"subnet": subnet})
 659
 660             if net_type == "data" and self.config.get('multisegment_support'):
 661                 if self.config.get('l2gw_support'):
 662                     l2gw_list = self.neutron.list_l2_gateways().get("l2_gateways", ())
 663                     for l2gw in l2gw_list:
 664                         l2gw_conn = {
 665                             "l2_gateway_id": l2gw["id"],
 666                             "network_id": new_net["network"]["id"],
 667                             "segmentation_id": str(vlanID),
 668                         }
 669                         new_l2gw_conn = self.neutron.create_l2_gateway_connection({"l2_gateway_connection": l2gw_conn})
 670                         created_items["l2gwconn:" + str(new_l2gw_conn["l2_gateway_connection"]["id"])] = True
 671             return new_net["network"]["id"], created_items
 672         except Exception as e:
 673             # delete l2gw connections (if any) before deleting the network
 674             for k, v in created_items.items():
 675                 if not v:  # skip already deleted
 676                     continue
 677                 try:
 678                     k_item, _, k_id = k.partition(":")
 679                     if k_item == "l2gwconn":
 680                         self.neutron.delete_l2_gateway_connection(k_id)
 681                 except Exception as e2:
 682                     self.logger.error("Error deleting l2 gateway connection: {}: {}".format(type(e2).__name__, e2))
 683             if new_net:
 684                 self.neutron.delete_network(new_net['network']['id'])
 685             self._format_exception(e)
 686
 687     def get_network_list(self, filter_dict={}):
 688         """Obtain tenant networks of VIM
 689         Filter_dict can be:
 690             name: network name
 691             id: network uuid
 692             shared: boolean
 693             tenant_id: tenant
 694             admin_state_up: boolean
 695             status: 'ACTIVE'
 696         Returns the network list of dictionaries
 697         """
 698         self.logger.debug("Getting network from VIM filter: '%s'", str(filter_dict))
 699         try:
 700             self._reload_connection()
 701             filter_dict_os = filter_dict.copy()
 702             if self.api_version3 and "tenant_id" in filter_dict_os:
 703                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')  # TODO check
 704             net_dict = self.neutron.list_networks(**filter_dict_os)
 705             net_list = net_dict["networks"]
 706             self.__net_os2mano(net_list)
 707             return net_list
 708         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException,
 709                 ConnectionError) as e:
 710             self._format_exception(e)
 711
 712     def get_network(self, net_id):
 713         """Obtain details of network from VIM
 714         Returns the network information from a network id"""
 715         self.logger.debug(" Getting tenant network %s from VIM", net_id)
 716         filter_dict = {"id": net_id}
 717         net_list = self.get_network_list(filter_dict)
 718         if len(net_list) == 0:
 719             raise vimconn.VimConnNotFoundException("Network '{}' not found".format(net_id))
 720         elif len(net_list) > 1:
 721             raise vimconn.VimConnConflictException("Found more than one network with this criteria")
 722         net = net_list[0]
 723         subnets = []
 724         for subnet_id in net.get("subnets", ()):
 725             try:
 726                 subnet = self.neutron.show_subnet(subnet_id)
 727             except Exception as e:
 728                 self.logger.error("osconnector.get_network(): Error getting subnet %s %s" % (net_id, str(e)))
 729                 subnet = {"id": subnet_id, "fault": str(e)}
 730             subnets.append(subnet)
 731         net["subnets"] = subnets
 732         net["encapsulation"] = net.get('provider:network_type')
 733         net["encapsulation_type"] = net.get('provider:network_type')
 734         net["segmentation_id"] = net.get('provider:segmentation_id')
 735         net["encapsulation_id"] = net.get('provider:segmentation_id')
 736         return net
 737
 738     def delete_network(self, net_id, created_items=None):
 739         """
 740         Removes a tenant network from VIM and its associated elements
 741         :param net_id: VIM identifier of the network, provided by method new_network
 742         :param created_items: dictionary with extra items to be deleted. provided by method new_network
 743         Returns the network identifier or raises an exception upon error or when network is not found
 744         """
 745         self.logger.debug("Deleting network '%s' from VIM", net_id)
 746         if created_items is None:
 747             created_items = {}
 748         try:
 749             self._reload_connection()
 750             # delete l2gw connections (if any) before deleting the network
 751             for k, v in created_items.items():
 752                 if not v:  # skip already deleted
 753                     continue
 754                 try:
 755                     k_item, _, k_id = k.partition(":")
 756                     if k_item == "l2gwconn":
 757                         self.neutron.delete_l2_gateway_connection(k_id)
 758                 except Exception as e:
 759                     self.logger.error("Error deleting l2 gateway connection: {}: {}".format(type(e).__name__, e))
 760             # delete VM ports attached to this networks before the network
 761             ports = self.neutron.list_ports(network_id=net_id)
 762             for p in ports['ports']:
 763                 try:
 764                     self.neutron.delete_port(p["id"])
 765                 except Exception as e:
 766                     self.logger.error("Error deleting port %s: %s", p["id"], str(e))
 767             self.neutron.delete_network(net_id)
 768             return net_id
 769         except (neExceptions.ConnectionFailed, neExceptions.NetworkNotFoundClient, neExceptions.NeutronException,
 770                 ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 771             self._format_exception(e)
 772
 773     def refresh_nets_status(self, net_list):
 774         """Get the status of the networks
 775            Params: the list of network identifiers
 776            Returns a dictionary with:
 777                 net_id:         #VIM id of this network
 778                     status:     #Mandatory. Text with one of:
 779                                 #  DELETED (not found at vim)
 780                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 781                                 #  OTHER (Vim reported other status not understood)
 782                                 #  ERROR (VIM indicates an ERROR status)
 783                                 #  ACTIVE, INACTIVE, DOWN (admin down),
 784                                 #  BUILD (on building process)
 785                                 #
 786                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 787                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 788
 789         """
 790         net_dict = {}
 791         for net_id in net_list:
 792             net = {}
 793             try:
 794                 net_vim = self.get_network(net_id)
 795                 if net_vim['status'] in netStatus2manoFormat:
 796                     net["status"] = netStatus2manoFormat[net_vim['status']]
 797                 else:
 798                     net["status"] = "OTHER"
 799                     net["error_msg"] = "VIM status reported " + net_vim['status']
 800
 801                 if net['status'] == "ACTIVE" and not net_vim['admin_state_up']:
 802                     net['status'] = 'DOWN'
 803
 804                 net['vim_info'] = self.serialize(net_vim)
 805
 806                 if net_vim.get('fault'):  # TODO
 807                     net['error_msg'] = str(net_vim['fault'])
 808             except vimconn.VimConnNotFoundException as e:
 809                 self.logger.error("Exception getting net status: %s", str(e))
 810                 net['status'] = "DELETED"
 811                 net['error_msg'] = str(e)
 812             except vimconn.VimConnException as e:
 813                 self.logger.error("Exception getting net status: %s", str(e))
 814                 net['status'] = "VIM_ERROR"
 815                 net['error_msg'] = str(e)
 816             net_dict[net_id] = net
 817         return net_dict
 818
 819     def get_flavor(self, flavor_id):
 820         """Obtain flavor details from the  VIM. Returns the flavor dict details"""
 821         self.logger.debug("Getting flavor '%s'", flavor_id)
 822         try:
 823             self._reload_connection()
 824             flavor = self.nova.flavors.find(id=flavor_id)
 825             # TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 826             return flavor.to_dict()
 827         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException,
 828                 ConnectionError) as e:
 829             self._format_exception(e)
 830
 831     def get_flavor_id_from_data(self, flavor_dict):
 832         """Obtain flavor id that match the flavor description
 833            Returns the flavor_id or raises a vimconnNotFoundException
 834            flavor_dict: contains the required ram, vcpus, disk
 835            If 'use_existing_flavors' is set to True at config, the closer flavor that provides same or more ram, vcpus
 836                 and disk is returned. Otherwise a flavor with exactly same ram, vcpus and disk is returned or a
 837                 vimconnNotFoundException is raised
 838         """
 839         exact_match = False if self.config.get('use_existing_flavors') else True
 840         try:
 841             self._reload_connection()
 842             flavor_candidate_id = None
 843             flavor_candidate_data = (10000, 10000, 10000)
 844             flavor_target = (flavor_dict["ram"], flavor_dict["vcpus"], flavor_dict["disk"])
 845             # numa=None
 846             extended = flavor_dict.get("extended", {})
 847             if extended:
 848                 # TODO
 849                 raise vimconn.VimConnNotFoundException("Flavor with EPA still not implemented")
 850                 # if len(numas) > 1:
 851                 #     raise vimconn.VimConnNotFoundException("Cannot find any flavor with more than one numa")
 852                 # numa=numas[0]
 853                 # numas = extended.get("numas")
 854             for flavor in self.nova.flavors.list():
 855                 epa = flavor.get_keys()
 856                 if epa:
 857                     continue
 858                     # TODO
 859                 flavor_data = (flavor.ram, flavor.vcpus, flavor.disk)
 860                 if flavor_data == flavor_target:
 861                     return flavor.id
 862                 elif not exact_match and flavor_target < flavor_data < flavor_candidate_data:
 863                     flavor_candidate_id = flavor.id
 864                     flavor_candidate_data = flavor_data
 865             if not exact_match and flavor_candidate_id:
 866                 return flavor_candidate_id
 867             raise vimconn.VimConnNotFoundException("Cannot find any flavor matching '{}'".format(flavor_dict))
 868         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException,
 869                 ConnectionError) as e:
 870             self._format_exception(e)
 871
 872     def process_resource_quota(self, quota, prefix, extra_specs):
 873         """
 874         :param prefix:
 875         :param extra_specs:
 876         :return:
 877         """
 878         if 'limit' in quota:
 879             extra_specs["quota:" + prefix + "_limit"] = quota['limit']
 880         if 'reserve' in quota:
 881             extra_specs["quota:" + prefix + "_reservation"] = quota['reserve']
 882         if 'shares' in quota:
 883             extra_specs["quota:" + prefix + "_shares_level"] = "custom"
 884             extra_specs["quota:" + prefix + "_shares_share"] = quota['shares']
 885
 886     def new_flavor(self, flavor_data, change_name_if_used=True):
 887         """Adds a tenant flavor to openstack VIM
 888         if change_name_if_used is True, it will change name in case of conflict, because it is not supported name
 889          repetition
 890         Returns the flavor identifier
 891         """
 892         self.logger.debug("Adding flavor '%s'", str(flavor_data))
 893         retry = 0
 894         max_retries = 3
 895         name_suffix = 0
 896         try:
 897             name = flavor_data['name']
 898             while retry < max_retries:
 899                 retry += 1
 900                 try:
 901                     self._reload_connection()
 902                     if change_name_if_used:
 903                         # get used names
 904                         fl_names = []
 905                         fl = self.nova.flavors.list()
 906                         for f in fl:
 907                             fl_names.append(f.name)
 908                         while name in fl_names:
 909                             name_suffix += 1
 910                             name = flavor_data['name']+"-" + str(name_suffix)
 911
 912                     ram = flavor_data.get('ram', 64)
 913                     vcpus = flavor_data.get('vcpus', 1)
 914                     extra_specs = {}
 915
 916                     extended = flavor_data.get("extended")
 917                     if extended:
 918                         numas = extended.get("numas")
 919                         if numas:
 920                             numa_nodes = len(numas)
 921                             if numa_nodes > 1:
 922                                 return -1, "Can not add flavor with more than one numa"
 923                             extra_specs["hw:numa_nodes"] = str(numa_nodes)
 924                             extra_specs["hw:mem_page_size"] = "large"
 925                             extra_specs["hw:cpu_policy"] = "dedicated"
 926                             extra_specs["hw:numa_mempolicy"] = "strict"
 927                             if self.vim_type == "VIO":
 928                                 extra_specs["vmware:extra_config"] = '{"numa.nodeAffinity":"0"}'
 929                                 extra_specs["vmware:latency_sensitivity_level"] = "high"
 930                             for numa in numas:
 931                                 # overwrite ram and vcpus
 932                                 # check if key 'memory' is present in numa else use ram value at flavor
 933                                 if 'memory' in numa:
 934                                     ram = numa['memory']*1024
 935                                 # See for reference: https://specs.openstack.org/openstack/nova-specs/specs/mitaka/
 936                                 # implemented/virt-driver-cpu-thread-pinning.html
 937                                 extra_specs["hw:cpu_sockets"] = 1
 938                                 if 'paired-threads' in numa:
 939                                     vcpus = numa['paired-threads']*2
 940                                     # cpu_thread_policy "require" implies that the compute node must have an
 941                                     # STM architecture
 942                                     extra_specs["hw:cpu_thread_policy"] = "require"
 943                                     extra_specs["hw:cpu_policy"] = "dedicated"
 944                                 elif 'cores' in numa:
 945                                     vcpus = numa['cores']
 946                                     # cpu_thread_policy "prefer" implies that the host must not have an SMT
 947                                     # architecture, or a non-SMT architecture will be emulated
 948                                     extra_specs["hw:cpu_thread_policy"] = "isolate"
 949                                     extra_specs["hw:cpu_policy"] = "dedicated"
 950                                 elif 'threads' in numa:
 951                                     vcpus = numa['threads']
 952                                     # cpu_thread_policy "prefer" implies that the host may or may not have an SMT
 953                                     # architecture
 954                                     extra_specs["hw:cpu_thread_policy"] = "prefer"
 955                                     extra_specs["hw:cpu_policy"] = "dedicated"
 956                                 # for interface in numa.get("interfaces",() ):
 957                                 #     if interface["dedicated"]=="yes":
 958                                 #         raise vimconn.VimConnException("Passthrough interfaces are not supported
 959                                 #         for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable)
 960                                 #     #TODO, add the key 'pci_passthrough:alias"="<label at config>:<number ifaces>"'
 961                                 #      when a way to connect it is available
 962                         elif extended.get("cpu-quota"):
 963                             self.process_resource_quota(extended.get("cpu-quota"), "cpu", extra_specs)
 964                         if extended.get("mem-quota"):
 965                             self.process_resource_quota(extended.get("mem-quota"), "memory", extra_specs)
 966                         if extended.get("vif-quota"):
 967                             self.process_resource_quota(extended.get("vif-quota"), "vif", extra_specs)
 968                         if extended.get("disk-io-quota"):
 969                             self.process_resource_quota(extended.get("disk-io-quota"), "disk_io", extra_specs)
 970                     # create flavor
 971                     new_flavor = self.nova.flavors.create(name,
 972                                                           ram,
 973                                                           vcpus,
 974                                                           flavor_data.get('disk', 0),
 975                                                           is_public=flavor_data.get('is_public', True)
 976                                                           )
 977                     # add metadata
 978                     if extra_specs:
 979                         new_flavor.set_keys(extra_specs)
 980                     return new_flavor.id
 981                 except nvExceptions.Conflict as e:
 982                     if change_name_if_used and retry < max_retries:
 983                         continue
 984                     self._format_exception(e)
 985         # except nvExceptions.BadRequest as e:
 986         except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError, KeyError) as e:
 987             self._format_exception(e)
 988
 989     def delete_flavor(self, flavor_id):
 990         """Deletes a tenant flavor from openstack VIM. Returns the old flavor_id
 991         """
 992         try:
 993             self._reload_connection()
 994             self.nova.flavors.delete(flavor_id)
 995             return flavor_id
 996         # except nvExceptions.BadRequest as e:
 997         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException,
 998                 ConnectionError) as e:
 999             self._format_exception(e)
1000
1001     def new_image(self, image_dict):
1002         """
1003         Adds a tenant image to VIM. imge_dict is a dictionary with:
1004             name: name
1005             disk_format: qcow2, vhd, vmdk, raw (by default), ...
1006             location: path or URI
1007             public: "yes" or "no"
1008             metadata: metadata of the image
1009         Returns the image_id
1010         """
1011         retry = 0
1012         max_retries = 3
1013         while retry < max_retries:
1014             retry += 1
1015             try:
1016                 self._reload_connection()
1017                 # determine format  http://docs.openstack.org/developer/glance/formats.html
1018                 if "disk_format" in image_dict:
1019                     disk_format = image_dict["disk_format"]
1020                 else:  # autodiscover based on extension
1021                     if image_dict['location'].endswith(".qcow2"):
1022                         disk_format = "qcow2"
1023                     elif image_dict['location'].endswith(".vhd"):
1024                         disk_format = "vhd"
1025                     elif image_dict['location'].endswith(".vmdk"):
1026                         disk_format = "vmdk"
1027                     elif image_dict['location'].endswith(".vdi"):
1028                         disk_format = "vdi"
1029                     elif image_dict['location'].endswith(".iso"):
1030                         disk_format = "iso"
1031                     elif image_dict['location'].endswith(".aki"):
1032                         disk_format = "aki"
1033                     elif image_dict['location'].endswith(".ari"):
1034                         disk_format = "ari"
1035                     elif image_dict['location'].endswith(".ami"):
1036                         disk_format = "ami"
1037                     else:
1038                         disk_format = "raw"
1039                 self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location'])
1040                 if self.vim_type == "VIO":
1041                     container_format = "bare"
1042                     if 'container_format' in image_dict:
1043                         container_format = image_dict['container_format']
1044                     new_image = self.glance.images.create(name=image_dict['name'], container_format=container_format,
1045                                                           disk_format=disk_format)
1046                 else:
1047                     new_image = self.glance.images.create(name=image_dict['name'])
1048                 if image_dict['location'].startswith("http"):
1049                     # TODO there is not a method to direct download. It must be downloaded locally with requests
1050                     raise vimconn.VimConnNotImplemented("Cannot create image from URL")
1051                 else:  # local path
1052                     with open(image_dict['location']) as fimage:
1053                         self.glance.images.upload(new_image.id, fimage)
1054                         # new_image = self.glancev1.images.create(name=image_dict['name'], is_public=
1055                         #  image_dict.get('public',"yes")=="yes",
1056                         #    container_format="bare", data=fimage, disk_format=disk_format)
1057                 metadata_to_load = image_dict.get('metadata')
1058                 # TODO location is a reserved word for current openstack versions. fixed for VIO please check
1059                 #  for openstack
1060                 if self.vim_type == "VIO":
1061                     metadata_to_load['upload_location'] = image_dict['location']
1062                 else:
1063                     metadata_to_load['location'] = image_dict['location']
1064                 self.glance.images.update(new_image.id, **metadata_to_load)
1065                 return new_image.id
1066             except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e:
1067                 self._format_exception(e)
1068             except (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError, ConnectionError) as e:
1069                 if retry == max_retries:
1070                     continue
1071                 self._format_exception(e)
1072             except IOError as e:  # can not open the file
1073                 raise vimconn.VimConnConnectionException("{}: {} for {}".format(type(e).__name__, e,
1074                                                                                 image_dict['location']),
1075                                                          http_code=vimconn.HTTP_Bad_Request)
1076
1077     def delete_image(self, image_id):
1078         """Deletes a tenant image from openstack VIM. Returns the old id
1079         """
1080         try:
1081             self._reload_connection()
1082             self.glance.images.delete(image_id)
1083             return image_id
1084         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException,
1085                 gl1Exceptions.CommunicationError, gl1Exceptions.HTTPNotFound, ConnectionError) as e:  # TODO remove
1086             self._format_exception(e)
1087
1088     def get_image_id_from_path(self, path):
1089         """Get the image id from image path in the VIM database. Returns the image_id"""
1090         try:
1091             self._reload_connection()
1092             images = self.glance.images.list()
1093             for image in images:
1094                 if image.metadata.get("location") == path:
1095                     return image.id
1096             raise vimconn.VimConnNotFoundException("image with location '{}' not found".format(path))
1097         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError,
1098                 ConnectionError) as e:
1099             self._format_exception(e)
1100
1101     def get_image_list(self, filter_dict={}):
1102         """Obtain tenant images from VIM
1103         Filter_dict can be:
1104             id: image id
1105             name: image name
1106             checksum: image checksum
1107         Returns the image list of dictionaries:
1108             [{<the fields at Filter_dict plus some VIM specific>}, ...]
1109             List can be empty
1110         """
1111         self.logger.debug("Getting image list from VIM filter: '%s'", str(filter_dict))
1112         try:
1113             self._reload_connection()
1114             # filter_dict_os = filter_dict.copy()
1115             # First we filter by the available filter fields: name, id. The others are removed.
1116             image_list = self.glance.images.list()
1117             filtered_list = []
1118             for image in image_list:
1119                 try:
1120                     if filter_dict.get("name") and image["name"] != filter_dict["name"]:
1121                         continue
1122                     if filter_dict.get("id") and image["id"] != filter_dict["id"]:
1123                         continue
1124                     if filter_dict.get("checksum") and image["checksum"] != filter_dict["checksum"]:
1125                         continue
1126
1127                     filtered_list.append(image.copy())
1128                 except gl1Exceptions.HTTPNotFound:
1129                     pass
1130             return filtered_list
1131         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError,
1132                 ConnectionError) as e:
1133             self._format_exception(e)
1134
1135     def __wait_for_vm(self, vm_id, status):
1136         """wait until vm is in the desired status and return True.
1137         If the VM gets in ERROR status, return false.
1138         If the timeout is reached generate an exception"""
1139         elapsed_time = 0
1140         while elapsed_time < server_timeout:
1141             vm_status = self.nova.servers.get(vm_id).status
1142             if vm_status == status:
1143                 return True
1144             if vm_status == 'ERROR':
1145                 return False
1146             time.sleep(5)
1147             elapsed_time += 5
1148
1149         # if we exceeded the timeout rollback
1150         if elapsed_time >= server_timeout:
1151             raise vimconn.VimConnException('Timeout waiting for instance ' + vm_id + ' to get ' + status,
1152                                            http_code=vimconn.HTTP_Request_Timeout)
1153
1154     def _get_openstack_availablity_zones(self):
1155         """
1156         Get from openstack availability zones available
1157         :return:
1158         """
1159         try:
1160             openstack_availability_zone = self.nova.availability_zones.list()
1161             openstack_availability_zone = [str(zone.zoneName) for zone in openstack_availability_zone
1162                                            if zone.zoneName != 'internal']
1163             return openstack_availability_zone
1164         except Exception:
1165             return None
1166
1167     def _set_availablity_zones(self):
1168         """
1169         Set vim availablity zone
1170         :return:
1171         """
1172
1173         if 'availability_zone' in self.config:
1174             vim_availability_zones = self.config.get('availability_zone')
1175             if isinstance(vim_availability_zones, str):
1176                 self.availability_zone = [vim_availability_zones]
1177             elif isinstance(vim_availability_zones, list):
1178                 self.availability_zone = vim_availability_zones
1179         else:
1180             self.availability_zone = self._get_openstack_availablity_zones()
1181
1182     def _get_vm_availability_zone(self, availability_zone_index, availability_zone_list):
1183         """
1184         Return thge availability zone to be used by the created VM.
1185         :return: The VIM availability zone to be used or None
1186         """
1187         if availability_zone_index is None:
1188             if not self.config.get('availability_zone'):
1189                 return None
1190             elif isinstance(self.config.get('availability_zone'), str):
1191                 return self.config['availability_zone']
1192             else:
1193                 # TODO consider using a different parameter at config for default AV and AV list match
1194                 return self.config['availability_zone'][0]
1195
1196         vim_availability_zones = self.availability_zone
1197         # check if VIM offer enough availability zones describe in the VNFD
1198         if vim_availability_zones and len(availability_zone_list) <= len(vim_availability_zones):
1199             # check if all the names of NFV AV match VIM AV names
1200             match_by_index = False
1201             for av in availability_zone_list:
1202                 if av not in vim_availability_zones:
1203                     match_by_index = True
1204                     break
1205             if match_by_index:
1206                 return vim_availability_zones[availability_zone_index]
1207             else:
1208                 return availability_zone_list[availability_zone_index]
1209         else:
1210             raise vimconn.VimConnConflictException("No enough availability zones at VIM for this deployment")
1211
1212     def new_vminstance(self, name, description, start, image_id, flavor_id, net_list, cloud_config=None, disk_list=None,
1213                        availability_zone_index=None, availability_zone_list=None):
1214         """Adds a VM instance to VIM
1215         Params:
1216             start: indicates if VM must start or boot in pause mode. Ignored
1217             image_id,flavor_id: iamge and flavor uuid
1218             net_list: list of interfaces, each one is a dictionary with:
1219                 name:
1220                 net_id: network uuid to connect
1221                 vpci: virtual vcpi to assign, ignored because openstack lack #TODO
1222                 model: interface model, ignored #TODO
1223                 mac_address: used for  SR-IOV ifaces #TODO for other types
1224                 use: 'data', 'bridge',  'mgmt'
1225                 type: 'virtual', 'PCI-PASSTHROUGH'('PF'), 'SR-IOV'('VF'), 'VFnotShared'
1226                 vim_id: filled/added by this function
1227                 floating_ip: True/False (or it can be None)
1228             'cloud_config': (optional) dictionary with:
1229                 'key-pairs': (optional) list of strings with the public key to be inserted to the default user
1230                 'users': (optional) list of users to be inserted, each item is a dict with:
1231                     'name': (mandatory) user name,
1232                     'key-pairs': (optional) list of strings with the public key to be inserted to the user
1233                 'user-data': (optional) string is a text script to be passed directly to cloud-init
1234                 'config-files': (optional). List of files to be transferred. Each item is a dict with:
1235                     'dest': (mandatory) string with the destination absolute path
1236                     'encoding': (optional, by default text). Can be one of:
1237                         'b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64'
1238                     'content' (mandatory): string with the content of the file
1239                     'permissions': (optional) string with file permissions, typically octal notation '0644'
1240                     'owner': (optional) file owner, string with the format 'owner:group'
1241                 'boot-data-drive': boolean to indicate if user-data must be passed using a boot drive (hard disk)
1242             'disk_list': (optional) list with additional disks to the VM. Each item is a dict with:
1243                 'image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted
1244                 'size': (mandatory) string with the size of the disk in GB
1245                 'vim_id' (optional) should use this existing volume id
1246             availability_zone_index: Index of availability_zone_list to use for this this VM. None if not AV required
1247             availability_zone_list: list of availability zones given by user in the VNFD descriptor.  Ignore if
1248                 availability_zone_index is None
1249                 #TODO ip, security groups
1250         Returns a tuple with the instance identifier and created_items or raises an exception on error
1251             created_items can be None or a dictionary where this method can include key-values that will be passed to
1252             the method delete_vminstance and action_vminstance. Can be used to store created ports, volumes, etc.
1253             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
1254             as not present.
1255         """
1256         self.logger.debug("new_vminstance input: image='%s' flavor='%s' nics='%s'", image_id, flavor_id, str(net_list))
1257         try:
1258             server = None
1259             created_items = {}
1260             # metadata = {}
1261             net_list_vim = []
1262             external_network = []
1263             # ^list of external networks to be connected to instance, later on used to create floating_ip
1264             no_secured_ports = []   # List of port-is with port-security disabled
1265             self._reload_connection()
1266             # metadata_vpci = {}   # For a specific neutron plugin
1267             block_device_mapping = None
1268
1269             for net in net_list:
1270                 if not net.get("net_id"):   # skip non connected iface
1271                     continue
1272
1273                 port_dict = {
1274                     "network_id": net["net_id"],
1275                     "name": net.get("name"),
1276                     "admin_state_up": True
1277                 }
1278                 if self.config.get("security_groups") and net.get("port_security") is not False and \
1279                         not self.config.get("no_port_security_extension"):
1280                     if not self.security_groups_id:
1281                         self._get_ids_from_name()
1282                     port_dict["security_groups"] = self.security_groups_id
1283
1284                 if net["type"] == "virtual":
1285                     pass
1286                     # if "vpci" in net:
1287                     #     metadata_vpci[ net["net_id"] ] = [[ net["vpci"], "" ]]
1288                 elif net["type"] == "VF" or net["type"] == "SR-IOV":  # for VF
1289                     # if "vpci" in net:
1290                     #     if "VF" not in metadata_vpci:
1291                     #         metadata_vpci["VF"]=[]
1292                     #     metadata_vpci["VF"].append([ net["vpci"], "" ])
1293                     port_dict["binding:vnic_type"] = "direct"
1294                     # VIO specific Changes
1295                     if self.vim_type == "VIO":
1296                         # Need to create port with port_security_enabled = False and no-security-groups
1297                         port_dict["port_security_enabled"] = False
1298                         port_dict["provider_security_groups"] = []
1299                         port_dict["security_groups"] = []
1300                 else:   # For PT PCI-PASSTHROUGH
1301                     # if "vpci" in net:
1302                     #     if "PF" not in metadata_vpci:
1303                     #         metadata_vpci["PF"]=[]
1304                     #     metadata_vpci["PF"].append([ net["vpci"], "" ])
1305                     port_dict["binding:vnic_type"] = "direct-physical"
1306                 if not port_dict["name"]:
1307                     port_dict["name"] = name
1308                 if net.get("mac_address"):
1309                     port_dict["mac_address"] = net["mac_address"]
1310                 if net.get("ip_address"):
1311                     port_dict["fixed_ips"] = [{'ip_address': net["ip_address"]}]
1312                     # TODO add 'subnet_id': <subnet_id>
1313                 new_port = self.neutron.create_port({"port": port_dict})
1314                 created_items["port:" + str(new_port["port"]["id"])] = True
1315                 net["mac_adress"] = new_port["port"]["mac_address"]
1316                 net["vim_id"] = new_port["port"]["id"]
1317                 # if try to use a network without subnetwork, it will return a emtpy list
1318                 fixed_ips = new_port["port"].get("fixed_ips")
1319                 if fixed_ips:
1320                     net["ip"] = fixed_ips[0].get("ip_address")
1321                 else:
1322                     net["ip"] = None
1323
1324                 port = {"port-id": new_port["port"]["id"]}
1325                 if float(self.nova.api_version.get_string()) >= 2.32:
1326                     port["tag"] = new_port["port"]["name"]
1327                 net_list_vim.append(port)
1328
1329                 if net.get('floating_ip', False):
1330                     net['exit_on_floating_ip_error'] = True
1331                     external_network.append(net)
1332                 elif net['use'] == 'mgmt' and self.config.get('use_floating_ip'):
1333                     net['exit_on_floating_ip_error'] = False
1334                     external_network.append(net)
1335                     net['floating_ip'] = self.config.get('use_floating_ip')
1336
1337                 # If port security is disabled when the port has not yet been attached to the VM, then all vm traffic
1338                 # is dropped.
1339                 # As a workaround we wait until the VM is active and then disable the port-security
1340                 if net.get("port_security") is False and not self.config.get("no_port_security_extension"):
1341                     no_secured_ports.append(new_port["port"]["id"])
1342
1343             # if metadata_vpci:
1344             #     metadata = {"pci_assignement": json.dumps(metadata_vpci)}
1345             #     if len(metadata["pci_assignement"]) >255:
1346             #         #limit the metadata size
1347             #         #metadata["pci_assignement"] = metadata["pci_assignement"][0:255]
1348             #         self.logger.warn("Metadata deleted since it exceeds the expected length (255) ")
1349             #         metadata = {}
1350
1351             self.logger.debug("name '%s' image_id '%s'flavor_id '%s' net_list_vim '%s' description '%s'",
1352                               name, image_id, flavor_id, str(net_list_vim), description)
1353
1354             # cloud config
1355             config_drive, userdata = self._create_user_data(cloud_config)
1356
1357             # Create additional volumes in case these are present in disk_list
1358             base_disk_index = ord('b')
1359             if disk_list:
1360                 block_device_mapping = {}
1361                 for disk in disk_list:
1362                     if disk.get('vim_id'):
1363                         block_device_mapping['_vd' + chr(base_disk_index)] = disk['vim_id']
1364                     else:
1365                         if 'image_id' in disk:
1366                             volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1367                                                                 chr(base_disk_index), imageRef=disk['image_id'])
1368                         else:
1369                             volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1370                                                                 chr(base_disk_index))
1371                         created_items["volume:" + str(volume.id)] = True
1372                         block_device_mapping['_vd' + chr(base_disk_index)] = volume.id
1373                     base_disk_index += 1
1374
1375                 # Wait until created volumes are with status available
1376                 elapsed_time = 0
1377                 while elapsed_time < volume_timeout:
1378                     for created_item in created_items:
1379                         v, _, volume_id = created_item.partition(":")
1380                         if v == 'volume':
1381                             if self.cinder.volumes.get(volume_id).status != 'available':
1382                                 break
1383                     else:  # all ready: break from while
1384                         break
1385                     time.sleep(5)
1386                     elapsed_time += 5
1387                 # If we exceeded the timeout rollback
1388                 if elapsed_time >= volume_timeout:
1389                     raise vimconn.VimConnException('Timeout creating volumes for instance ' + name,
1390                                                    http_code=vimconn.HTTP_Request_Timeout)
1391             # get availability Zone
1392             vm_av_zone = self._get_vm_availability_zone(availability_zone_index, availability_zone_list)
1393
1394             self.logger.debug("nova.servers.create({}, {}, {}, nics={}, security_groups={}, "
1395                               "availability_zone={}, key_name={}, userdata={}, config_drive={}, "
1396                               "block_device_mapping={})".format(name, image_id, flavor_id, net_list_vim,
1397                                                                 self.config.get("security_groups"), vm_av_zone,
1398                                                                 self.config.get('keypair'), userdata, config_drive,
1399                                                                 block_device_mapping))
1400             server = self.nova.servers.create(name, image_id, flavor_id, nics=net_list_vim,
1401                                               security_groups=self.config.get("security_groups"),
1402                                               # TODO remove security_groups in future versions. Already at neutron port
1403                                               availability_zone=vm_av_zone,
1404                                               key_name=self.config.get('keypair'),
1405                                               userdata=userdata,
1406                                               config_drive=config_drive,
1407                                               block_device_mapping=block_device_mapping
1408                                               )  # , description=description)
1409
1410             vm_start_time = time.time()
1411             # Previously mentioned workaround to wait until the VM is active and then disable the port-security
1412             if no_secured_ports:
1413                 self.__wait_for_vm(server.id, 'ACTIVE')
1414
1415             for port_id in no_secured_ports:
1416                 try:
1417                     self.neutron.update_port(port_id,
1418                                              {"port": {"port_security_enabled": False, "security_groups": None}})
1419                 except Exception:
1420                     raise vimconn.VimConnException("It was not possible to disable port security for port {}".format(
1421                         port_id))
1422             # print "DONE :-)", server
1423
1424             # pool_id = None
1425             for floating_network in external_network:
1426                 try:
1427                     assigned = False
1428                     floating_ip_retries = 3
1429                     # In case of RO in HA there can be conflicts, two RO trying to assign same floating IP, so retry
1430                     # several times
1431                     while not assigned:
1432                         floating_ips = self.neutron.list_floatingips().get("floatingips", ())
1433                         random.shuffle(floating_ips)   # randomize
1434                         for fip in floating_ips:
1435                             if fip.get("port_id") or fip.get('tenant_id') != server.tenant_id:
1436                                 continue
1437                             if isinstance(floating_network['floating_ip'], str):
1438                                 if fip.get("floating_network_id") != floating_network['floating_ip']:
1439                                     continue
1440                             free_floating_ip = fip["id"]
1441                             break
1442                         else:
1443                             if isinstance(floating_network['floating_ip'], str) and \
1444                                     floating_network['floating_ip'].lower() != "true":
1445                                 pool_id = floating_network['floating_ip']
1446                             else:
1447                                 # Find the external network
1448                                 external_nets = list()
1449                                 for net in self.neutron.list_networks()['networks']:
1450                                     if net['router:external']:
1451                                         external_nets.append(net)
1452
1453                                 if len(external_nets) == 0:
1454                                     raise vimconn.VimConnException(
1455                                         "Cannot create floating_ip automatically since no external network is present",
1456                                         http_code=vimconn.HTTP_Conflict)
1457                                 if len(external_nets) > 1:
1458                                     raise vimconn.VimConnException(
1459                                         "Cannot create floating_ip automatically since multiple external networks are"
1460                                         " present", http_code=vimconn.HTTP_Conflict)
1461
1462                                 pool_id = external_nets[0].get('id')
1463                             param = {'floatingip': {'floating_network_id': pool_id, 'tenant_id': server.tenant_id}}
1464                             try:
1465                                 # self.logger.debug("Creating floating IP")
1466                                 new_floating_ip = self.neutron.create_floatingip(param)
1467                                 free_floating_ip = new_floating_ip['floatingip']['id']
1468                                 created_items["floating_ip:" + str(free_floating_ip)] = True
1469                             except Exception as e:
1470                                 raise vimconn.VimConnException(type(e).__name__ + ": Cannot create new floating_ip " +
1471                                                                str(e), http_code=vimconn.HTTP_Conflict)
1472
1473                         try:
1474                             # for race condition ensure not already assigned
1475                             fip = self.neutron.show_floatingip(free_floating_ip)
1476                             if fip['floatingip']['port_id']:
1477                                 continue
1478                             # the vim_id key contains the neutron.port_id
1479                             self.neutron.update_floatingip(free_floating_ip,
1480                                                            {"floatingip": {"port_id": floating_network["vim_id"]}})
1481                             # for race condition ensure not re-assigned to other VM after 5 seconds
1482                             time.sleep(5)
1483                             fip = self.neutron.show_floatingip(free_floating_ip)
1484                             if fip['floatingip']['port_id'] != floating_network["vim_id"]:
1485                                 self.logger.error("floating_ip {} re-assigned to other port".format(free_floating_ip))
1486                                 continue
1487                             self.logger.debug("Assigned floating_ip {} to VM {}".format(free_floating_ip, server.id))
1488                             assigned = True
1489                         except Exception as e:
1490                             # openstack need some time after VM creation to assign an IP. So retry if fails
1491                             vm_status = self.nova.servers.get(server.id).status
1492                             if vm_status not in ('ACTIVE', 'ERROR'):
1493                                 if time.time() - vm_start_time < server_timeout:
1494                                     time.sleep(5)
1495                                     continue
1496                             elif floating_ip_retries > 0:
1497                                 floating_ip_retries -= 1
1498                                 continue
1499                             raise vimconn.VimConnException(
1500                                 "Cannot create floating_ip: {} {}".format(type(e).__name__, e),
1501                                 http_code=vimconn.HTTP_Conflict)
1502
1503                 except Exception as e:
1504                     if not floating_network['exit_on_floating_ip_error']:
1505                         self.logger.error("Cannot create floating_ip. %s", str(e))
1506                         continue
1507                     raise
1508
1509             return server.id, created_items
1510         # except nvExceptions.NotFound as e:
1511         #     error_value=-vimconn.HTTP_Not_Found
1512         #     error_text= "vm instance %s not found" % vm_id
1513         # except TypeError as e:
1514         #     raise vimconn.VimConnException(type(e).__name__ + ": "+  str(e), http_code=vimconn.HTTP_Bad_Request)
1515
1516         except Exception as e:
1517             server_id = None
1518             if server:
1519                 server_id = server.id
1520             try:
1521                 self.delete_vminstance(server_id, created_items)
1522             except Exception as e2:
1523                 self.logger.error("new_vminstance rollback fail {}".format(e2))
1524
1525             self._format_exception(e)
1526
1527     def get_vminstance(self, vm_id):
1528         """Returns the VM instance information from VIM"""
1529         # self.logger.debug("Getting VM from VIM")
1530         try:
1531             self._reload_connection()
1532             server = self.nova.servers.find(id=vm_id)
1533             # TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
1534             return server.to_dict()
1535         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound,
1536                 ConnectionError) as e:
1537             self._format_exception(e)
1538
1539     def get_vminstance_console(self, vm_id, console_type="vnc"):
1540         """
1541         Get a console for the virtual machine
1542         Params:
1543             vm_id: uuid of the VM
1544             console_type, can be:
1545                 "novnc" (by default), "xvpvnc" for VNC types,
1546                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
1547         Returns dict with the console parameters:
1548                 protocol: ssh, ftp, http, https, ...
1549                 server:   usually ip address
1550                 port:     the http, ssh, ... port
1551                 suffix:   extra text, e.g. the http path and query string
1552         """
1553         self.logger.debug("Getting VM CONSOLE from VIM")
1554         try:
1555             self._reload_connection()
1556             server = self.nova.servers.find(id=vm_id)
1557             if console_type is None or console_type == "novnc":
1558                 console_dict = server.get_vnc_console("novnc")
1559             elif console_type == "xvpvnc":
1560                 console_dict = server.get_vnc_console(console_type)
1561             elif console_type == "rdp-html5":
1562                 console_dict = server.get_rdp_console(console_type)
1563             elif console_type == "spice-html5":
1564                 console_dict = server.get_spice_console(console_type)
1565             else:
1566                 raise vimconn.VimConnException("console type '{}' not allowed".format(console_type),
1567                                                http_code=vimconn.HTTP_Bad_Request)
1568
1569             console_dict1 = console_dict.get("console")
1570             if console_dict1:
1571                 console_url = console_dict1.get("url")
1572                 if console_url:
1573                     # parse console_url
1574                     protocol_index = console_url.find("//")
1575                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1576                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1577                     if protocol_index < 0 or port_index < 0 or suffix_index < 0:
1578                         return -vimconn.HTTP_Internal_Server_Error, "Unexpected response from VIM"
1579                     console_dict = {"protocol": console_url[0:protocol_index],
1580                                     "server": console_url[protocol_index+2:port_index],
1581                                     "port": console_url[port_index:suffix_index],
1582                                     "suffix": console_url[suffix_index+1:]
1583                                     }
1584                     protocol_index += 2
1585                     return console_dict
1586             raise vimconn.VimConnUnexpectedResponse("Unexpected response from VIM")
1587
1588         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException,
1589                 nvExceptions.BadRequest, ConnectionError) as e:
1590             self._format_exception(e)
1591
1592     def delete_vminstance(self, vm_id, created_items=None):
1593         """Removes a VM instance from VIM. Returns the old identifier
1594         """
1595         # print "osconnector: Getting VM from VIM"
1596         if created_items is None:
1597             created_items = {}
1598         try:
1599             self._reload_connection()
1600             # delete VM ports attached to this networks before the virtual machine
1601             for k, v in created_items.items():
1602                 if not v:  # skip already deleted
1603                     continue
1604                 try:
1605                     k_item, _, k_id = k.partition(":")
1606                     if k_item == "port":
1607                         self.neutron.delete_port(k_id)
1608                 except Exception as e:
1609                     self.logger.error("Error deleting port: {}: {}".format(type(e).__name__, e))
1610
1611             # #commented because detaching the volumes makes the servers.delete not work properly ?!?
1612             # #dettach volumes attached
1613             # server = self.nova.servers.get(vm_id)
1614             # volumes_attached_dict = server._info['os-extended-volumes:volumes_attached']   #volume['id']
1615             # #for volume in volumes_attached_dict:
1616             # #    self.cinder.volumes.detach(volume['id'])
1617
1618             if vm_id:
1619                 self.nova.servers.delete(vm_id)
1620
1621             # delete volumes. Although having detached, they should have in active status before deleting
1622             # we ensure in this loop
1623             keep_waiting = True
1624             elapsed_time = 0
1625             while keep_waiting and elapsed_time < volume_timeout:
1626                 keep_waiting = False
1627                 for k, v in created_items.items():
1628                     if not v:  # skip already deleted
1629                         continue
1630                     try:
1631                         k_item, _, k_id = k.partition(":")
1632                         if k_item == "volume":
1633                             if self.cinder.volumes.get(k_id).status != 'available':
1634                                 keep_waiting = True
1635                             else:
1636                                 self.cinder.volumes.delete(k_id)
1637                                 created_items[k] = None
1638                         elif k_item == "floating_ip":  # floating ip
1639                             self.neutron.delete_floatingip(k_id)
1640                             created_items[k] = None
1641
1642                     except Exception as e:
1643                         self.logger.error("Error deleting {}: {}".format(k, e))
1644                 if keep_waiting:
1645                     time.sleep(1)
1646                     elapsed_time += 1
1647             return None
1648         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException,
1649                 ConnectionError) as e:
1650             self._format_exception(e)
1651
1652     def refresh_vms_status(self, vm_list):
1653         """Get the status of the virtual machines and their interfaces/ports
1654            Params: the list of VM identifiers
1655            Returns a dictionary with:
1656                 vm_id:          #VIM id of this Virtual Machine
1657                     status:     #Mandatory. Text with one of:
1658                                 #  DELETED (not found at vim)
1659                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1660                                 #  OTHER (Vim reported other status not understood)
1661                                 #  ERROR (VIM indicates an ERROR status)
1662                                 #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
1663                                 #  CREATING (on building process), ERROR
1664                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
1665                                 #
1666                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
1667                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
1668                     interfaces:
1669                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
1670                         mac_address:      #Text format XX:XX:XX:XX:XX:XX
1671                         vim_net_id:       #network id where this interface is connected
1672                         vim_interface_id: #interface/port VIM id
1673                         ip_address:       #null, or text with IPv4, IPv6 address
1674                         compute_node:     #identification of compute node where PF,VF interface is allocated
1675                         pci:              #PCI address of the NIC that hosts the PF,VF
1676                         vlan:             #physical VLAN used for VF
1677         """
1678         vm_dict = {}
1679         self.logger.debug("refresh_vms status: Getting tenant VM instance information from VIM")
1680         for vm_id in vm_list:
1681             vm = {}
1682             try:
1683                 vm_vim = self.get_vminstance(vm_id)
1684                 if vm_vim['status'] in vmStatus2manoFormat:
1685                     vm['status'] = vmStatus2manoFormat[vm_vim['status']]
1686                 else:
1687                     vm['status'] = "OTHER"
1688                     vm['error_msg'] = "VIM status reported " + vm_vim['status']
1689
1690                 vm['vim_info'] = self.serialize(vm_vim)
1691
1692                 vm["interfaces"] = []
1693                 if vm_vim.get('fault'):
1694                     vm['error_msg'] = str(vm_vim['fault'])
1695                 # get interfaces
1696                 try:
1697                     self._reload_connection()
1698                     port_dict = self.neutron.list_ports(device_id=vm_id)
1699                     for port in port_dict["ports"]:
1700                         interface = {}
1701                         interface['vim_info'] = self.serialize(port)
1702                         interface["mac_address"] = port.get("mac_address")
1703                         interface["vim_net_id"] = port["network_id"]
1704                         interface["vim_interface_id"] = port["id"]
1705                         # check if OS-EXT-SRV-ATTR:host is there,
1706                         # in case of non-admin credentials, it will be missing
1707                         if vm_vim.get('OS-EXT-SRV-ATTR:host'):
1708                             interface["compute_node"] = vm_vim['OS-EXT-SRV-ATTR:host']
1709                         interface["pci"] = None
1710
1711                         # check if binding:profile is there,
1712                         # in case of non-admin credentials, it will be missing
1713                         if port.get('binding:profile'):
1714                             if port['binding:profile'].get('pci_slot'):
1715                                 # TODO: At the moment sr-iov pci addresses are converted to PF pci addresses by setting
1716                                 #  the slot to 0x00
1717                                 # TODO: This is just a workaround valid for niantinc. Find a better way to do so
1718                                 #   CHANGE DDDD:BB:SS.F to DDDD:BB:00.(F%2)   assuming there are 2 ports per nic
1719                                 pci = port['binding:profile']['pci_slot']
1720                                 # interface["pci"] = pci[:-4] + "00." + str(int(pci[-1]) % 2)
1721                                 interface["pci"] = pci
1722                         interface["vlan"] = None
1723                         if port.get('binding:vif_details'):
1724                             interface["vlan"] = port['binding:vif_details'].get('vlan')
1725                         # Get vlan from network in case not present in port for those old openstacks and cases where
1726                         # it is needed vlan at PT
1727                         if not interface["vlan"]:
1728                             # if network is of type vlan and port is of type direct (sr-iov) then set vlan id
1729                             network = self.neutron.show_network(port["network_id"])
1730                             if network['network'].get('provider:network_type') == 'vlan':
1731                                 # and port.get("binding:vnic_type") in ("direct", "direct-physical"):
1732                                 interface["vlan"] = network['network'].get('provider:segmentation_id')
1733                         ips = []
1734                         # look for floating ip address
1735                         try:
1736                             floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
1737                             if floating_ip_dict.get("floatingips"):
1738                                 ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address"))
1739                         except Exception:
1740                             pass
1741
1742                         for subnet in port["fixed_ips"]:
1743                             ips.append(subnet["ip_address"])
1744                         interface["ip_address"] = ";".join(ips)
1745                         vm["interfaces"].append(interface)
1746                 except Exception as e:
1747                     self.logger.error("Error getting vm interface information {}: {}".format(type(e).__name__, e),
1748                                       exc_info=True)
1749             except vimconn.VimConnNotFoundException as e:
1750                 self.logger.error("Exception getting vm status: %s", str(e))
1751                 vm['status'] = "DELETED"
1752                 vm['error_msg'] = str(e)
1753             except vimconn.VimConnException as e:
1754                 self.logger.error("Exception getting vm status: %s", str(e))
1755                 vm['status'] = "VIM_ERROR"
1756                 vm['error_msg'] = str(e)
1757             vm_dict[vm_id] = vm
1758         return vm_dict
1759
1760     def action_vminstance(self, vm_id, action_dict, created_items={}):
1761         """Send and action over a VM instance from VIM
1762         Returns None or the console dict if the action was successfully sent to the VIM"""
1763         self.logger.debug("Action over VM '%s': %s", vm_id, str(action_dict))
1764         try:
1765             self._reload_connection()
1766             server = self.nova.servers.find(id=vm_id)
1767             if "start" in action_dict:
1768                 if action_dict["start"] == "rebuild":
1769                     server.rebuild()
1770                 else:
1771                     if server.status == "PAUSED":
1772                         server.unpause()
1773                     elif server.status == "SUSPENDED":
1774                         server.resume()
1775                     elif server.status == "SHUTOFF":
1776                         server.start()
1777             elif "pause" in action_dict:
1778                 server.pause()
1779             elif "resume" in action_dict:
1780                 server.resume()
1781             elif "shutoff" in action_dict or "shutdown" in action_dict:
1782                 server.stop()
1783             elif "forceOff" in action_dict:
1784                 server.stop()  # TODO
1785             elif "terminate" in action_dict:
1786                 server.delete()
1787             elif "createImage" in action_dict:
1788                 server.create_image()
1789                 # "path":path_schema,
1790                 # "description":description_schema,
1791                 # "name":name_schema,
1792                 # "metadata":metadata_schema,
1793                 # "imageRef": id_schema,
1794                 # "disk": {"oneOf":[{"type": "null"}, {"type":"string"}] },
1795             elif "rebuild" in action_dict:
1796                 server.rebuild(server.image['id'])
1797             elif "reboot" in action_dict:
1798                 server.reboot()   # reboot_type='SOFT'
1799             elif "console" in action_dict:
1800                 console_type = action_dict["console"]
1801                 if console_type is None or console_type == "novnc":
1802                     console_dict = server.get_vnc_console("novnc")
1803                 elif console_type == "xvpvnc":
1804                     console_dict = server.get_vnc_console(console_type)
1805                 elif console_type == "rdp-html5":
1806                     console_dict = server.get_rdp_console(console_type)
1807                 elif console_type == "spice-html5":
1808                     console_dict = server.get_spice_console(console_type)
1809                 else:
1810                     raise vimconn.VimConnException("console type '{}' not allowed".format(console_type),
1811                                                    http_code=vimconn.HTTP_Bad_Request)
1812                 try:
1813                     console_url = console_dict["console"]["url"]
1814                     # parse console_url
1815                     protocol_index = console_url.find("//")
1816                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1817                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1818                     if protocol_index < 0 or port_index < 0 or suffix_index < 0:
1819                         raise vimconn.VimConnException("Unexpected response from VIM " + str(console_dict))
1820                     console_dict2 = {"protocol": console_url[0:protocol_index],
1821                                      "server": console_url[protocol_index+2: port_index],
1822                                      "port": int(console_url[port_index+1: suffix_index]),
1823                                      "suffix": console_url[suffix_index+1:]
1824                                      }
1825                     return console_dict2
1826                 except Exception:
1827                     raise vimconn.VimConnException("Unexpected response from VIM " + str(console_dict))
1828
1829             return None
1830         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound,
1831                 ConnectionError) as e:
1832             self._format_exception(e)
1833         # TODO insert exception vimconn.HTTP_Unauthorized
1834
1835     # ###### VIO Specific Changes #########
1836     def _generate_vlanID(self):
1837         """
1838          Method to get unused vlanID
1839             Args:
1840                 None
1841             Returns:
1842                 vlanID
1843         """
1844         # Get used VLAN IDs
1845         usedVlanIDs = []
1846         networks = self.get_network_list()
1847         for net in networks:
1848             if net.get('provider:segmentation_id'):
1849                 usedVlanIDs.append(net.get('provider:segmentation_id'))
1850         used_vlanIDs = set(usedVlanIDs)
1851
1852         # find unused VLAN ID
1853         for vlanID_range in self.config.get('dataplane_net_vlan_range'):
1854             try:
1855                 start_vlanid, end_vlanid = map(int, vlanID_range.replace(" ", "").split("-"))
1856                 for vlanID in range(start_vlanid, end_vlanid + 1):
1857                     if vlanID not in used_vlanIDs:
1858                         return vlanID
1859             except Exception as exp:
1860                 raise vimconn.VimConnException("Exception {} occurred while generating VLAN ID.".format(exp))
1861         else:
1862             raise vimconn.VimConnConflictException(
1863                 "Unable to create the SRIOV VLAN network. All given Vlan IDs {} are in use.".format(
1864                     self.config.get('dataplane_net_vlan_range')))
1865
1866     def _generate_multisegment_vlanID(self):
1867         """
1868          Method to get unused vlanID
1869             Args:
1870                 None
1871             Returns:
1872                 vlanID
1873         """
1874         # Get used VLAN IDs
1875         usedVlanIDs = []
1876         networks = self.get_network_list()
1877         for net in networks:
1878             if net.get('provider:network_type') == "vlan" and net.get('provider:segmentation_id'):
1879                 usedVlanIDs.append(net.get('provider:segmentation_id'))
1880             elif net.get('segments'):
1881                 for segment in net.get('segments'):
1882                     if segment.get('provider:network_type') == "vlan" and segment.get('provider:segmentation_id'):
1883                         usedVlanIDs.append(segment.get('provider:segmentation_id'))
1884         used_vlanIDs = set(usedVlanIDs)
1885
1886         # find unused VLAN ID
1887         for vlanID_range in self.config.get('multisegment_vlan_range'):
1888             try:
1889                 start_vlanid, end_vlanid = map(int, vlanID_range.replace(" ", "").split("-"))
1890                 for vlanID in range(start_vlanid, end_vlanid + 1):
1891                     if vlanID not in used_vlanIDs:
1892                         return vlanID
1893             except Exception as exp:
1894                 raise vimconn.VimConnException("Exception {} occurred while generating VLAN ID.".format(exp))
1895         else:
1896             raise vimconn.VimConnConflictException(
1897                 "Unable to create the VLAN segment. All VLAN IDs {} are in use.".format(
1898                     self.config.get('multisegment_vlan_range')))
1899
1900     def _validate_vlan_ranges(self, input_vlan_range, text_vlan_range):
1901         """
1902         Method to validate user given vlanID ranges
1903             Args:  None
1904             Returns: None
1905         """
1906         for vlanID_range in input_vlan_range:
1907             vlan_range = vlanID_range.replace(" ", "")
1908             # validate format
1909             vlanID_pattern = r'(\d)*-(\d)*$'
1910             match_obj = re.match(vlanID_pattern, vlan_range)
1911             if not match_obj:
1912                 raise vimconn.VimConnConflictException(
1913                     "Invalid VLAN range for {}: {}.You must provide '{}' in format [start_ID - end_ID].".format(
1914                         text_vlan_range, vlanID_range, text_vlan_range))
1915
1916             start_vlanid, end_vlanid = map(int, vlan_range.split("-"))
1917             if start_vlanid <= 0:
1918                 raise vimconn.VimConnConflictException(
1919                     "Invalid VLAN range for {}: {}. Start ID can not be zero. For VLAN "
1920                     "networks valid IDs are 1 to 4094 ".format(text_vlan_range, vlanID_range))
1921             if end_vlanid > 4094:
1922                 raise vimconn.VimConnConflictException(
1923                     "Invalid VLAN range for {}: {}. End VLAN ID can not be greater than 4094. For VLAN "
1924                     "networks valid IDs are 1 to 4094 ".format(text_vlan_range, vlanID_range))
1925
1926             if start_vlanid > end_vlanid:
1927                 raise vimconn.VimConnConflictException(
1928                     "Invalid VLAN range for {}: {}. You must provide '{}' in format start_ID - end_ID and "
1929                     "start_ID < end_ID ".format(text_vlan_range, vlanID_range, text_vlan_range))
1930
1931     # NOT USED FUNCTIONS
1932
1933     def new_external_port(self, port_data):
1934         """Adds a external port to VIM
1935            Returns the port identifier"""
1936         # TODO openstack if needed
1937         return -vimconn.HTTP_Internal_Server_Error, "osconnector.new_external_port() not implemented"
1938
1939     def connect_port_network(self, port_id, network_id, admin=False):
1940         """Connects a external port to a network
1941             Returns status code of the VIM response"""
1942         # TODO openstack if needed
1943         return -vimconn.HTTP_Internal_Server_Error, "osconnector.connect_port_network() not implemented"
1944
1945     def new_user(self, user_name, user_passwd, tenant_id=None):
1946         """Adds a new user to openstack VIM
1947            Returns the user identifier"""
1948         self.logger.debug("osconnector: Adding a new user to VIM")
1949         try:
1950             self._reload_connection()
1951             user = self.keystone.users.create(user_name, password=user_passwd, default_project=tenant_id)
1952             # self.keystone.tenants.add_user(self.k_creds["username"], #role)
1953             return user.id
1954         except ksExceptions.ConnectionError as e:
1955             error_value = -vimconn.HTTP_Bad_Request
1956             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
1957         except ksExceptions.ClientException as e:  # TODO remove
1958             error_value = -vimconn.HTTP_Bad_Request
1959             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
1960         # TODO insert exception vimconn.HTTP_Unauthorized
1961         # if reaching here is because an exception
1962         self.logger.debug("new_user " + error_text)
1963         return error_value, error_text
1964
1965     def delete_user(self, user_id):
1966         """Delete a user from openstack VIM
1967            Returns the user identifier"""
1968         if self.debug:
1969             print("osconnector: Deleting  a  user from VIM")
1970         try:
1971             self._reload_connection()
1972             self.keystone.users.delete(user_id)
1973             return 1, user_id
1974         except ksExceptions.ConnectionError as e:
1975             error_value = -vimconn.HTTP_Bad_Request
1976             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
1977         except ksExceptions.NotFound as e:
1978             error_value = -vimconn.HTTP_Not_Found
1979             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
1980         except ksExceptions.ClientException as e:  # TODO remove
1981             error_value = -vimconn.HTTP_Bad_Request
1982             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
1983         # TODO insert exception vimconn.HTTP_Unauthorized
1984         # if reaching here is because an exception
1985         self.logger.debug("delete_tenant " + error_text)
1986         return error_value, error_text
1987
1988     def get_hosts_info(self):
1989         """Get the information of deployed hosts
1990         Returns the hosts content"""
1991         if self.debug:
1992             print("osconnector: Getting Host info from VIM")
1993         try:
1994             h_list = []
1995             self._reload_connection()
1996             hypervisors = self.nova.hypervisors.list()
1997             for hype in hypervisors:
1998                 h_list.append(hype.to_dict())
1999             return 1, {"hosts": h_list}
2000         except nvExceptions.NotFound as e:
2001             error_value = -vimconn.HTTP_Not_Found
2002             error_text = (str(e) if len(e.args) == 0 else str(e.args[0]))
2003         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
2004             error_value = -vimconn.HTTP_Bad_Request
2005             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
2006         # TODO insert exception vimconn.HTTP_Unauthorized
2007         # if reaching here is because an exception
2008         self.logger.debug("get_hosts_info " + error_text)
2009         return error_value, error_text
2010
2011     def get_hosts(self, vim_tenant):
2012         """Get the hosts and deployed instances
2013         Returns the hosts content"""
2014         r, hype_dict = self.get_hosts_info()
2015         if r < 0:
2016             return r, hype_dict
2017         hypervisors = hype_dict["hosts"]
2018         try:
2019             servers = self.nova.servers.list()
2020             for hype in hypervisors:
2021                 for server in servers:
2022                     if server.to_dict()['OS-EXT-SRV-ATTR:hypervisor_hostname'] == hype['hypervisor_hostname']:
2023                         if 'vm' in hype:
2024                             hype['vm'].append(server.id)
2025                         else:
2026                             hype['vm'] = [server.id]
2027             return 1, hype_dict
2028         except nvExceptions.NotFound as e:
2029             error_value = -vimconn.HTTP_Not_Found
2030             error_text = (str(e) if len(e.args) == 0 else str(e.args[0]))
2031         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
2032             error_value = -vimconn.HTTP_Bad_Request
2033             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
2034         # TODO insert exception vimconn.HTTP_Unauthorized
2035         # if reaching here is because an exception
2036         self.logger.debug("get_hosts " + error_text)
2037         return error_value, error_text
2038
2039     def new_classification(self, name, ctype, definition):
2040         self.logger.debug('Adding a new (Traffic) Classification to VIM, named %s', name)
2041         try:
2042             new_class = None
2043             self._reload_connection()
2044             if ctype not in supportedClassificationTypes:
2045                 raise vimconn.VimConnNotSupportedException(
2046                     'OpenStack VIM connector does not support provided Classification Type {}, supported ones are: '
2047                     '{}'.format(ctype, supportedClassificationTypes))
2048             if not self._validate_classification(ctype, definition):
2049                 raise vimconn.VimConnException(
2050                     'Incorrect Classification definition '
2051                     'for the type specified.')
2052             classification_dict = definition
2053             classification_dict['name'] = name
2054
2055             new_class = self.neutron.create_sfc_flow_classifier(
2056                 {'flow_classifier': classification_dict})
2057             return new_class['flow_classifier']['id']
2058         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2059                 neExceptions.NeutronException, ConnectionError) as e:
2060             self.logger.error(
2061                 'Creation of Classification failed.')
2062             self._format_exception(e)
2063
2064     def get_classification(self, class_id):
2065         self.logger.debug(" Getting Classification %s from VIM", class_id)
2066         filter_dict = {"id": class_id}
2067         class_list = self.get_classification_list(filter_dict)
2068         if len(class_list) == 0:
2069             raise vimconn.VimConnNotFoundException(
2070                 "Classification '{}' not found".format(class_id))
2071         elif len(class_list) > 1:
2072             raise vimconn.VimConnConflictException(
2073                 "Found more than one Classification with this criteria")
2074         classification = class_list[0]
2075         return classification
2076
2077     def get_classification_list(self, filter_dict={}):
2078         self.logger.debug("Getting Classifications from VIM filter: '%s'",
2079                           str(filter_dict))
2080         try:
2081             filter_dict_os = filter_dict.copy()
2082             self._reload_connection()
2083             if self.api_version3 and "tenant_id" in filter_dict_os:
2084                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
2085             classification_dict = self.neutron.list_sfc_flow_classifiers(
2086                 **filter_dict_os)
2087             classification_list = classification_dict["flow_classifiers"]
2088             self.__classification_os2mano(classification_list)
2089             return classification_list
2090         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2091                 neExceptions.NeutronException, ConnectionError) as e:
2092             self._format_exception(e)
2093
2094     def delete_classification(self, class_id):
2095         self.logger.debug("Deleting Classification '%s' from VIM", class_id)
2096         try:
2097             self._reload_connection()
2098             self.neutron.delete_sfc_flow_classifier(class_id)
2099             return class_id
2100         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
2101                 ksExceptions.ClientException, neExceptions.NeutronException,
2102                 ConnectionError) as e:
2103             self._format_exception(e)
2104
2105     def new_sfi(self, name, ingress_ports, egress_ports, sfc_encap=True):
2106         self.logger.debug("Adding a new Service Function Instance to VIM, named '%s'", name)
2107         try:
2108             new_sfi = None
2109             self._reload_connection()
2110             correlation = None
2111             if sfc_encap:
2112                 correlation = 'nsh'
2113             if len(ingress_ports) != 1:
2114                 raise vimconn.VimConnNotSupportedException(
2115                     "OpenStack VIM connector can only have "
2116                     "1 ingress port per SFI")
2117             if len(egress_ports) != 1:
2118                 raise vimconn.VimConnNotSupportedException(
2119                     "OpenStack VIM connector can only have "
2120                     "1 egress port per SFI")
2121             sfi_dict = {'name': name,
2122                         'ingress': ingress_ports[0],
2123                         'egress': egress_ports[0],
2124                         'service_function_parameters': {
2125                             'correlation': correlation}}
2126             new_sfi = self.neutron.create_sfc_port_pair({'port_pair': sfi_dict})
2127             return new_sfi['port_pair']['id']
2128         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2129                 neExceptions.NeutronException, ConnectionError) as e:
2130             if new_sfi:
2131                 try:
2132                     self.neutron.delete_sfc_port_pair(
2133                         new_sfi['port_pair']['id'])
2134                 except Exception:
2135                     self.logger.error(
2136                         'Creation of Service Function Instance failed, with '
2137                         'subsequent deletion failure as well.')
2138             self._format_exception(e)
2139
2140     def get_sfi(self, sfi_id):
2141         self.logger.debug('Getting Service Function Instance %s from VIM', sfi_id)
2142         filter_dict = {"id": sfi_id}
2143         sfi_list = self.get_sfi_list(filter_dict)
2144         if len(sfi_list) == 0:
2145             raise vimconn.VimConnNotFoundException("Service Function Instance '{}' not found".format(sfi_id))
2146         elif len(sfi_list) > 1:
2147             raise vimconn.VimConnConflictException(
2148                 'Found more than one Service Function Instance '
2149                 'with this criteria')
2150         sfi = sfi_list[0]
2151         return sfi
2152
2153     def get_sfi_list(self, filter_dict={}):
2154         self.logger.debug("Getting Service Function Instances from VIM filter: '%s'", str(filter_dict))
2155         try:
2156             self._reload_connection()
2157             filter_dict_os = filter_dict.copy()
2158             if self.api_version3 and "tenant_id" in filter_dict_os:
2159                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
2160             sfi_dict = self.neutron.list_sfc_port_pairs(**filter_dict_os)
2161             sfi_list = sfi_dict["port_pairs"]
2162             self.__sfi_os2mano(sfi_list)
2163             return sfi_list
2164         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2165                 neExceptions.NeutronException, ConnectionError) as e:
2166             self._format_exception(e)
2167
2168     def delete_sfi(self, sfi_id):
2169         self.logger.debug("Deleting Service Function Instance '%s' "
2170                           "from VIM", sfi_id)
2171         try:
2172             self._reload_connection()
2173             self.neutron.delete_sfc_port_pair(sfi_id)
2174             return sfi_id
2175         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
2176                 ksExceptions.ClientException, neExceptions.NeutronException,
2177                 ConnectionError) as e:
2178             self._format_exception(e)
2179
2180     def new_sf(self, name, sfis, sfc_encap=True):
2181         self.logger.debug("Adding a new Service Function to VIM, named '%s'", name)
2182         try:
2183             new_sf = None
2184             self._reload_connection()
2185             # correlation = None
2186             # if sfc_encap:
2187             #     correlation = 'nsh'
2188             for instance in sfis:
2189                 sfi = self.get_sfi(instance)
2190                 if sfi.get('sfc_encap') != sfc_encap:
2191                     raise vimconn.VimConnNotSupportedException(
2192                         "OpenStack VIM connector requires all SFIs of the "
2193                         "same SF to share the same SFC Encapsulation")
2194             sf_dict = {'name': name,
2195                        'port_pairs': sfis}
2196             new_sf = self.neutron.create_sfc_port_pair_group({
2197                 'port_pair_group': sf_dict})
2198             return new_sf['port_pair_group']['id']
2199         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2200                 neExceptions.NeutronException, ConnectionError) as e:
2201             if new_sf:
2202                 try:
2203                     self.neutron.delete_sfc_port_pair_group(
2204                         new_sf['port_pair_group']['id'])
2205                 except Exception:
2206                     self.logger.error(
2207                         'Creation of Service Function failed, with '
2208                         'subsequent deletion failure as well.')
2209             self._format_exception(e)
2210
2211     def get_sf(self, sf_id):
2212         self.logger.debug("Getting Service Function %s from VIM", sf_id)
2213         filter_dict = {"id": sf_id}
2214         sf_list = self.get_sf_list(filter_dict)
2215         if len(sf_list) == 0:
2216             raise vimconn.VimConnNotFoundException(
2217                 "Service Function '{}' not found".format(sf_id))
2218         elif len(sf_list) > 1:
2219             raise vimconn.VimConnConflictException(
2220                 "Found more than one Service Function with this criteria")
2221         sf = sf_list[0]
2222         return sf
2223
2224     def get_sf_list(self, filter_dict={}):
2225         self.logger.debug("Getting Service Function from VIM filter: '%s'",
2226                           str(filter_dict))
2227         try:
2228             self._reload_connection()
2229             filter_dict_os = filter_dict.copy()
2230             if self.api_version3 and "tenant_id" in filter_dict_os:
2231                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
2232             sf_dict = self.neutron.list_sfc_port_pair_groups(**filter_dict_os)
2233             sf_list = sf_dict["port_pair_groups"]
2234             self.__sf_os2mano(sf_list)
2235             return sf_list
2236         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2237                 neExceptions.NeutronException, ConnectionError) as e:
2238             self._format_exception(e)
2239
2240     def delete_sf(self, sf_id):
2241         self.logger.debug("Deleting Service Function '%s' from VIM", sf_id)
2242         try:
2243             self._reload_connection()
2244             self.neutron.delete_sfc_port_pair_group(sf_id)
2245             return sf_id
2246         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
2247                 ksExceptions.ClientException, neExceptions.NeutronException,
2248                 ConnectionError) as e:
2249             self._format_exception(e)
2250
2251     def new_sfp(self, name, classifications, sfs, sfc_encap=True, spi=None):
2252         self.logger.debug("Adding a new Service Function Path to VIM, named '%s'", name)
2253         try:
2254             new_sfp = None
2255             self._reload_connection()
2256             # In networking-sfc the MPLS encapsulation is legacy
2257             # should be used when no full SFC Encapsulation is intended
2258             correlation = 'mpls'
2259             if sfc_encap:
2260                 correlation = 'nsh'
2261             sfp_dict = {'name': name,
2262                         'flow_classifiers': classifications,
2263                         'port_pair_groups': sfs,
2264                         'chain_parameters': {'correlation': correlation}}
2265             if spi:
2266                 sfp_dict['chain_id'] = spi
2267             new_sfp = self.neutron.create_sfc_port_chain({'port_chain': sfp_dict})
2268             return new_sfp["port_chain"]["id"]
2269         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2270                 neExceptions.NeutronException, ConnectionError) as e:
2271             if new_sfp:
2272                 try:
2273                     self.neutron.delete_sfc_port_chain(new_sfp['port_chain']['id'])
2274                 except Exception:
2275                     self.logger.error(
2276                         'Creation of Service Function Path failed, with '
2277                         'subsequent deletion failure as well.')
2278             self._format_exception(e)
2279
2280     def get_sfp(self, sfp_id):
2281         self.logger.debug(" Getting Service Function Path %s from VIM", sfp_id)
2282         filter_dict = {"id": sfp_id}
2283         sfp_list = self.get_sfp_list(filter_dict)
2284         if len(sfp_list) == 0:
2285             raise vimconn.VimConnNotFoundException(
2286                 "Service Function Path '{}' not found".format(sfp_id))
2287         elif len(sfp_list) > 1:
2288             raise vimconn.VimConnConflictException(
2289                 "Found more than one Service Function Path with this criteria")
2290         sfp = sfp_list[0]
2291         return sfp
2292
2293     def get_sfp_list(self, filter_dict={}):
2294         self.logger.debug("Getting Service Function Paths from VIM filter: '%s'", str(filter_dict))
2295         try:
2296             self._reload_connection()
2297             filter_dict_os = filter_dict.copy()
2298             if self.api_version3 and "tenant_id" in filter_dict_os:
2299                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
2300             sfp_dict = self.neutron.list_sfc_port_chains(**filter_dict_os)
2301             sfp_list = sfp_dict["port_chains"]
2302             self.__sfp_os2mano(sfp_list)
2303             return sfp_list
2304         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2305                 neExceptions.NeutronException, ConnectionError) as e:
2306             self._format_exception(e)
2307
2308     def delete_sfp(self, sfp_id):
2309         self.logger.debug("Deleting Service Function Path '%s' from VIM", sfp_id)
2310         try:
2311             self._reload_connection()
2312             self.neutron.delete_sfc_port_chain(sfp_id)
2313             return sfp_id
2314         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
2315                 ksExceptions.ClientException, neExceptions.NeutronException,
2316                 ConnectionError) as e:
2317             self._format_exception(e)
2318
2319     def refresh_sfps_status(self, sfp_list):
2320         """Get the status of the service function path
2321            Params: the list of sfp identifiers
2322            Returns a dictionary with:
2323                 vm_id:          #VIM id of this service function path
2324                     status:     #Mandatory. Text with one of:
2325                                 #  DELETED (not found at vim)
2326                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
2327                                 #  OTHER (Vim reported other status not understood)
2328                                 #  ERROR (VIM indicates an ERROR status)
2329                                 #  ACTIVE,
2330                                 #  CREATING (on building process)
2331                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
2332                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)F
2333         """
2334         sfp_dict = {}
2335         self.logger.debug("refresh_sfps status: Getting tenant SFP information from VIM")
2336         for sfp_id in sfp_list:
2337             sfp = {}
2338             try:
2339                 sfp_vim = self.get_sfp(sfp_id)
2340                 if sfp_vim['spi']:
2341                     sfp['status'] = vmStatus2manoFormat['ACTIVE']
2342                 else:
2343                     sfp['status'] = "OTHER"
2344                     sfp['error_msg'] = "VIM status reported " + sfp['status']
2345
2346                 sfp['vim_info'] = self.serialize(sfp_vim)
2347
2348                 if sfp_vim.get('fault'):
2349                     sfp['error_msg'] = str(sfp_vim['fault'])
2350
2351             except vimconn.VimConnNotFoundException as e:
2352                 self.logger.error("Exception getting sfp status: %s", str(e))
2353                 sfp['status'] = "DELETED"
2354                 sfp['error_msg'] = str(e)
2355             except vimconn.VimConnException as e:
2356                 self.logger.error("Exception getting sfp status: %s", str(e))
2357                 sfp['status'] = "VIM_ERROR"
2358                 sfp['error_msg'] = str(e)
2359             sfp_dict[sfp_id] = sfp
2360         return sfp_dict
2361
2362     def refresh_sfis_status(self, sfi_list):
2363         """Get the status of the service function instances
2364            Params: the list of sfi identifiers
2365            Returns a dictionary with:
2366                 vm_id:          #VIM id of this service function instance
2367                     status:     #Mandatory. Text with one of:
2368                                 #  DELETED (not found at vim)
2369                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
2370                                 #  OTHER (Vim reported other status not understood)
2371                                 #  ERROR (VIM indicates an ERROR status)
2372                                 #  ACTIVE,
2373                                 #  CREATING (on building process)
2374                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
2375                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
2376         """
2377         sfi_dict = {}
2378         self.logger.debug("refresh_sfis status: Getting tenant sfi information from VIM")
2379         for sfi_id in sfi_list:
2380             sfi = {}
2381             try:
2382                 sfi_vim = self.get_sfi(sfi_id)
2383                 if sfi_vim:
2384                     sfi['status'] = vmStatus2manoFormat['ACTIVE']
2385                 else:
2386                     sfi['status'] = "OTHER"
2387                     sfi['error_msg'] = "VIM status reported " + sfi['status']
2388
2389                 sfi['vim_info'] = self.serialize(sfi_vim)
2390
2391                 if sfi_vim.get('fault'):
2392                     sfi['error_msg'] = str(sfi_vim['fault'])
2393
2394             except vimconn.VimConnNotFoundException as e:
2395                 self.logger.error("Exception getting sfi status: %s", str(e))
2396                 sfi['status'] = "DELETED"
2397                 sfi['error_msg'] = str(e)
2398             except vimconn.VimConnException as e:
2399                 self.logger.error("Exception getting sfi status: %s", str(e))
2400                 sfi['status'] = "VIM_ERROR"
2401                 sfi['error_msg'] = str(e)
2402             sfi_dict[sfi_id] = sfi
2403         return sfi_dict
2404
2405     def refresh_sfs_status(self, sf_list):
2406         """Get the status of the service functions
2407            Params: the list of sf identifiers
2408            Returns a dictionary with:
2409                 vm_id:          #VIM id of this service function
2410                     status:     #Mandatory. Text with one of:
2411                                 #  DELETED (not found at vim)
2412                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
2413                                 #  OTHER (Vim reported other status not understood)
2414                                 #  ERROR (VIM indicates an ERROR status)
2415                                 #  ACTIVE,
2416                                 #  CREATING (on building process)
2417                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
2418                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
2419         """
2420         sf_dict = {}
2421         self.logger.debug("refresh_sfs status: Getting tenant sf information from VIM")
2422         for sf_id in sf_list:
2423             sf = {}
2424             try:
2425                 sf_vim = self.get_sf(sf_id)
2426                 if sf_vim:
2427                     sf['status'] = vmStatus2manoFormat['ACTIVE']
2428                 else:
2429                     sf['status'] = "OTHER"
2430                     sf['error_msg'] = "VIM status reported " + sf_vim['status']
2431
2432                 sf['vim_info'] = self.serialize(sf_vim)
2433
2434                 if sf_vim.get('fault'):
2435                     sf['error_msg'] = str(sf_vim['fault'])
2436
2437             except vimconn.VimConnNotFoundException as e:
2438                 self.logger.error("Exception getting sf status: %s", str(e))
2439                 sf['status'] = "DELETED"
2440                 sf['error_msg'] = str(e)
2441             except vimconn.VimConnException as e:
2442                 self.logger.error("Exception getting sf status: %s", str(e))
2443                 sf['status'] = "VIM_ERROR"
2444                 sf['error_msg'] = str(e)
2445             sf_dict[sf_id] = sf
2446         return sf_dict
2447
2448     def refresh_classifications_status(self, classification_list):
2449         """Get the status of the classifications
2450            Params: the list of classification identifiers
2451            Returns a dictionary with:
2452                 vm_id:          #VIM id of this classifier
2453                     status:     #Mandatory. Text with one of:
2454                                 #  DELETED (not found at vim)
2455                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
2456                                 #  OTHER (Vim reported other status not understood)
2457                                 #  ERROR (VIM indicates an ERROR status)
2458                                 #  ACTIVE,
2459                                 #  CREATING (on building process)
2460                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
2461                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
2462         """
2463         classification_dict = {}
2464         self.logger.debug("refresh_classifications status: Getting tenant classification information from VIM")
2465         for classification_id in classification_list:
2466             classification = {}
2467             try:
2468                 classification_vim = self.get_classification(classification_id)
2469                 if classification_vim:
2470                     classification['status'] = vmStatus2manoFormat['ACTIVE']
2471                 else:
2472                     classification['status'] = "OTHER"
2473                     classification['error_msg'] = "VIM status reported " + classification['status']
2474
2475                 classification['vim_info'] = self.serialize(classification_vim)
2476
2477                 if classification_vim.get('fault'):
2478                     classification['error_msg'] = str(classification_vim['fault'])
2479
2480             except vimconn.VimConnNotFoundException as e:
2481                 self.logger.error("Exception getting classification status: %s", str(e))
2482                 classification['status'] = "DELETED"
2483                 classification['error_msg'] = str(e)
2484             except vimconn.VimConnException as e:
2485                 self.logger.error("Exception getting classification status: %s", str(e))
2486                 classification['status'] = "VIM_ERROR"
2487                 classification['error_msg'] = str(e)
2488             classification_dict[classification_id] = classification
2489         return classification_dict