RO-VIM-openstack/osm_rovim_openstack/vimconn_openstack.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 ##
  20
  21 """
  22 osconnector implements all the methods to interact with openstack using the python-neutronclient.
  23
  24 For the VNF forwarding graph, The OpenStack VIM connector calls the
  25 networking-sfc Neutron extension methods, whose resources are mapped
  26 to the VIM connector's SFC resources as follows:
  27 - Classification (OSM) -> Flow Classifier (Neutron)
  28 - Service Function Instance (OSM) -> Port Pair (Neutron)
  29 - Service Function (OSM) -> Port Pair Group (Neutron)
  30 - Service Function Path (OSM) -> Port Chain (Neutron)
  31 """
  32
  33 from osm_ro_plugin import vimconn
  34 # import json
  35 import logging
  36 import netaddr
  37 import time
  38 import yaml
  39 import random
  40 import re
  41 import copy
  42 from pprint import pformat
  43
  44 from novaclient import client as nClient, exceptions as nvExceptions
  45 from keystoneauth1.identity import v2, v3
  46 from keystoneauth1 import session
  47 import keystoneclient.exceptions as ksExceptions
  48 import keystoneclient.v3.client as ksClient_v3
  49 import keystoneclient.v2_0.client as ksClient_v2
  50 from glanceclient import client as glClient
  51 import glanceclient.exc as gl1Exceptions
  52 from cinderclient import client as cClient
  53 from http.client import HTTPException   # TODO py3 check that this base exception matches python2 httplib.HTTPException
  54 from neutronclient.neutron import client as neClient
  55 from neutronclient.common import exceptions as neExceptions
  56 from requests.exceptions import ConnectionError
  57
  58 __author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes, xFlow Research, Igor D.C., Eduardo Sousa"
  59 __date__ = "$22-sep-2017 23:59:59$"
  60
  61 """contain the openstack virtual machine status to openmano status"""
  62 vmStatus2manoFormat = {'ACTIVE': 'ACTIVE',
  63                        'PAUSED': 'PAUSED',
  64                        'SUSPENDED': 'SUSPENDED',
  65                        'SHUTOFF': 'INACTIVE',
  66                        'BUILD': 'BUILD',
  67                        'ERROR': 'ERROR',
  68                        'DELETED': 'DELETED'
  69                        }
  70 netStatus2manoFormat = {'ACTIVE': 'ACTIVE',
  71                         'PAUSED': 'PAUSED',
  72                         'INACTIVE': 'INACTIVE',
  73                         'BUILD': 'BUILD',
  74                         'ERROR': 'ERROR',
  75                         'DELETED': 'DELETED'
  76                         }
  77
  78 supportedClassificationTypes = ['legacy_flow_classifier']
  79
  80 # global var to have a timeout creating and deleting volumes
  81 volume_timeout = 1800
  82 server_timeout = 1800
  83
  84
  85 class SafeDumper(yaml.SafeDumper):
  86     def represent_data(self, data):
  87         # Openstack APIs use custom subclasses of dict and YAML safe dumper
  88         # is designed to not handle that (reference issue 142 of pyyaml)
  89         if isinstance(data, dict) and data.__class__ != dict:
  90             # A simple solution is to convert those items back to dicts
  91             data = dict(data.items())
  92
  93         return super(SafeDumper, self).represent_data(data)
  94
  95
  96 class vimconnector(vimconn.VimConnector):
  97     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None,
  98                  log_level=None, config={}, persistent_info={}):
  99         """using common constructor parameters. In this case
 100         'url' is the keystone authorization url,
 101         'url_admin' is not use
 102         """
 103         api_version = config.get('APIversion')
 104         if api_version and api_version not in ('v3.3', 'v2.0', '2', '3'):
 105             raise vimconn.VimConnException("Invalid value '{}' for config:APIversion. "
 106                                            "Allowed values are 'v3.3', 'v2.0', '2' or '3'".format(api_version))
 107         vim_type = config.get('vim_type')
 108         if vim_type and vim_type not in ('vio', 'VIO'):
 109             raise vimconn.VimConnException("Invalid value '{}' for config:vim_type."
 110                                            "Allowed values are 'vio' or 'VIO'".format(vim_type))
 111
 112         if config.get('dataplane_net_vlan_range') is not None:
 113             # validate vlan ranges provided by user
 114             self._validate_vlan_ranges(config.get('dataplane_net_vlan_range'), 'dataplane_net_vlan_range')
 115
 116         if config.get('multisegment_vlan_range') is not None:
 117             # validate vlan ranges provided by user
 118             self._validate_vlan_ranges(config.get('multisegment_vlan_range'), 'multisegment_vlan_range')
 119
 120         vimconn.VimConnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level,
 121                                       config)
 122
 123         if self.config.get("insecure") and self.config.get("ca_cert"):
 124             raise vimconn.VimConnException("options insecure and ca_cert are mutually exclusive")
 125         self.verify = True
 126         if self.config.get("insecure"):
 127             self.verify = False
 128         if self.config.get("ca_cert"):
 129             self.verify = self.config.get("ca_cert")
 130
 131         if not url:
 132             raise TypeError('url param can not be NoneType')
 133         self.persistent_info = persistent_info
 134         self.availability_zone = persistent_info.get('availability_zone', None)
 135         self.session = persistent_info.get('session', {'reload_client': True})
 136         self.my_tenant_id = self.session.get('my_tenant_id')
 137         self.nova = self.session.get('nova')
 138         self.neutron = self.session.get('neutron')
 139         self.cinder = self.session.get('cinder')
 140         self.glance = self.session.get('glance')
 141         # self.glancev1 = self.session.get('glancev1')
 142         self.keystone = self.session.get('keystone')
 143         self.api_version3 = self.session.get('api_version3')
 144         self.vim_type = self.config.get("vim_type")
 145         if self.vim_type:
 146             self.vim_type = self.vim_type.upper()
 147         if self.config.get("use_internal_endpoint"):
 148             self.endpoint_type = "internalURL"
 149         else:
 150             self.endpoint_type = None
 151
 152         logging.getLogger('urllib3').setLevel(logging.WARNING)
 153         logging.getLogger('keystoneauth').setLevel(logging.WARNING)
 154         logging.getLogger('novaclient').setLevel(logging.WARNING)
 155         self.logger = logging.getLogger('ro.vim.openstack')
 156
 157         # allow security_groups to be a list or a single string
 158         if isinstance(self.config.get('security_groups'), str):
 159             self.config['security_groups'] = [self.config['security_groups']]
 160         self.security_groups_id = None
 161
 162         # ###### VIO Specific Changes #########
 163         if self.vim_type == "VIO":
 164             self.logger = logging.getLogger('ro.vim.vio')
 165
 166         if log_level:
 167             self.logger.setLevel(getattr(logging, log_level))
 168
 169     def __getitem__(self, index):
 170         """Get individuals parameters.
 171         Throw KeyError"""
 172         if index == 'project_domain_id':
 173             return self.config.get("project_domain_id")
 174         elif index == 'user_domain_id':
 175             return self.config.get("user_domain_id")
 176         else:
 177             return vimconn.VimConnector.__getitem__(self, index)
 178
 179     def __setitem__(self, index, value):
 180         """Set individuals parameters and it is marked as dirty so to force connection reload.
 181         Throw KeyError"""
 182         if index == 'project_domain_id':
 183             self.config["project_domain_id"] = value
 184         elif index == 'user_domain_id':
 185             self.config["user_domain_id"] = value
 186         else:
 187             vimconn.VimConnector.__setitem__(self, index, value)
 188         self.session['reload_client'] = True
 189
 190     def serialize(self, value):
 191         """Serialization of python basic types.
 192
 193         In the case value is not serializable a message will be logged and a
 194         simple representation of the data that cannot be converted back to
 195         python is returned.
 196         """
 197         if isinstance(value, str):
 198             return value
 199
 200         try:
 201             return yaml.dump(value, Dumper=SafeDumper,
 202                              default_flow_style=True, width=256)
 203         except yaml.representer.RepresenterError:
 204             self.logger.debug('The following entity cannot be serialized in YAML:\n\n%s\n\n', pformat(value),
 205                               exc_info=True)
 206             return str(value)
 207
 208     def _reload_connection(self):
 209         """Called before any operation, it check if credentials has changed
 210         Throw keystoneclient.apiclient.exceptions.AuthorizationFailure
 211         """
 212         # TODO control the timing and possible token timeout, but it seams that python client does this task for us :-)
 213         if self.session['reload_client']:
 214             if self.config.get('APIversion'):
 215                 self.api_version3 = self.config['APIversion'] == 'v3.3' or self.config['APIversion'] == '3'
 216             else:  # get from ending auth_url that end with v3 or with v2.0
 217                 self.api_version3 = self.url.endswith("/v3") or self.url.endswith("/v3/")
 218             self.session['api_version3'] = self.api_version3
 219             if self.api_version3:
 220                 if self.config.get('project_domain_id') or self.config.get('project_domain_name'):
 221                     project_domain_id_default = None
 222                 else:
 223                     project_domain_id_default = 'default'
 224                 if self.config.get('user_domain_id') or self.config.get('user_domain_name'):
 225                     user_domain_id_default = None
 226                 else:
 227                     user_domain_id_default = 'default'
 228                 auth = v3.Password(auth_url=self.url,
 229                                    username=self.user,
 230                                    password=self.passwd,
 231                                    project_name=self.tenant_name,
 232                                    project_id=self.tenant_id,
 233                                    project_domain_id=self.config.get('project_domain_id', project_domain_id_default),
 234                                    user_domain_id=self.config.get('user_domain_id', user_domain_id_default),
 235                                    project_domain_name=self.config.get('project_domain_name'),
 236                                    user_domain_name=self.config.get('user_domain_name'))
 237             else:
 238                 auth = v2.Password(auth_url=self.url,
 239                                    username=self.user,
 240                                    password=self.passwd,
 241                                    tenant_name=self.tenant_name,
 242                                    tenant_id=self.tenant_id)
 243             sess = session.Session(auth=auth, verify=self.verify)
 244             # addedd region_name to keystone, nova, neutron and cinder to support distributed cloud for Wind River
 245             # Titanium cloud and StarlingX
 246             region_name = self.config.get('region_name')
 247             if self.api_version3:
 248                 self.keystone = ksClient_v3.Client(session=sess, endpoint_type=self.endpoint_type,
 249                                                    region_name=region_name)
 250             else:
 251                 self.keystone = ksClient_v2.Client(session=sess, endpoint_type=self.endpoint_type)
 252             self.session['keystone'] = self.keystone
 253             # In order to enable microversion functionality an explicit microversion must be specified in 'config'.
 254             # This implementation approach is due to the warning message in
 255             # https://developer.openstack.org/api-guide/compute/microversions.html
 256             # where it is stated that microversion backwards compatibility is not guaranteed and clients should
 257             # always require an specific microversion.
 258             # To be able to use 'device role tagging' functionality define 'microversion: 2.32' in datacenter config
 259             version = self.config.get("microversion")
 260             if not version:
 261                 version = "2.1"
 262             # addedd region_name to keystone, nova, neutron and cinder to support distributed cloud for Wind River
 263             # Titanium cloud and StarlingX
 264             self.nova = self.session['nova'] = nClient.Client(str(version), session=sess,
 265                                                               endpoint_type=self.endpoint_type, region_name=region_name)
 266             self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess,
 267                                                                      endpoint_type=self.endpoint_type,
 268                                                                      region_name=region_name)
 269             self.cinder = self.session['cinder'] = cClient.Client(2, session=sess, endpoint_type=self.endpoint_type,
 270                                                                   region_name=region_name)
 271             try:
 272                 self.my_tenant_id = self.session['my_tenant_id'] = sess.get_project_id()
 273             except Exception:
 274                 self.logger.error("Cannot get project_id from session", exc_info=True)
 275             if self.endpoint_type == "internalURL":
 276                 glance_service_id = self.keystone.services.list(name="glance")[0].id
 277                 glance_endpoint = self.keystone.endpoints.list(glance_service_id, interface="internal")[0].url
 278             else:
 279                 glance_endpoint = None
 280             self.glance = self.session['glance'] = glClient.Client(2, session=sess, endpoint=glance_endpoint)
 281             # using version 1 of glance client in new_image()
 282             # self.glancev1 = self.session['glancev1'] = glClient.Client('1', session=sess,
 283             #                                                            endpoint=glance_endpoint)
 284             self.session['reload_client'] = False
 285             self.persistent_info['session'] = self.session
 286             # add availablity zone info inside  self.persistent_info
 287             self._set_availablity_zones()
 288             self.persistent_info['availability_zone'] = self.availability_zone
 289             self.security_groups_id = None  # force to get again security_groups_ids next time they are needed
 290
 291     def __net_os2mano(self, net_list_dict):
 292         """Transform the net openstack format to mano format
 293         net_list_dict can be a list of dict or a single dict"""
 294         if type(net_list_dict) is dict:
 295             net_list_ = (net_list_dict,)
 296         elif type(net_list_dict) is list:
 297             net_list_ = net_list_dict
 298         else:
 299             raise TypeError("param net_list_dict must be a list or a dictionary")
 300         for net in net_list_:
 301             if net.get('provider:network_type') == "vlan":
 302                 net['type'] = 'data'
 303             else:
 304                 net['type'] = 'bridge'
 305
 306     def __classification_os2mano(self, class_list_dict):
 307         """Transform the openstack format (Flow Classifier) to mano format
 308         (Classification) class_list_dict can be a list of dict or a single dict
 309         """
 310         if isinstance(class_list_dict, dict):
 311             class_list_ = [class_list_dict]
 312         elif isinstance(class_list_dict, list):
 313             class_list_ = class_list_dict
 314         else:
 315             raise TypeError("param class_list_dict must be a list or a dictionary")
 316         for classification in class_list_:
 317             id = classification.pop('id')
 318             name = classification.pop('name')
 319             description = classification.pop('description')
 320             project_id = classification.pop('project_id')
 321             tenant_id = classification.pop('tenant_id')
 322             original_classification = copy.deepcopy(classification)
 323             classification.clear()
 324             classification['ctype'] = 'legacy_flow_classifier'
 325             classification['definition'] = original_classification
 326             classification['id'] = id
 327             classification['name'] = name
 328             classification['description'] = description
 329             classification['project_id'] = project_id
 330             classification['tenant_id'] = tenant_id
 331
 332     def __sfi_os2mano(self, sfi_list_dict):
 333         """Transform the openstack format (Port Pair) to mano format (SFI)
 334         sfi_list_dict can be a list of dict or a single dict
 335         """
 336         if isinstance(sfi_list_dict, dict):
 337             sfi_list_ = [sfi_list_dict]
 338         elif isinstance(sfi_list_dict, list):
 339             sfi_list_ = sfi_list_dict
 340         else:
 341             raise TypeError(
 342                 "param sfi_list_dict must be a list or a dictionary")
 343         for sfi in sfi_list_:
 344             sfi['ingress_ports'] = []
 345             sfi['egress_ports'] = []
 346             if sfi.get('ingress'):
 347                 sfi['ingress_ports'].append(sfi['ingress'])
 348             if sfi.get('egress'):
 349                 sfi['egress_ports'].append(sfi['egress'])
 350             del sfi['ingress']
 351             del sfi['egress']
 352             params = sfi.get('service_function_parameters')
 353             sfc_encap = False
 354             if params:
 355                 correlation = params.get('correlation')
 356                 if correlation:
 357                     sfc_encap = True
 358             sfi['sfc_encap'] = sfc_encap
 359             del sfi['service_function_parameters']
 360
 361     def __sf_os2mano(self, sf_list_dict):
 362         """Transform the openstack format (Port Pair Group) to mano format (SF)
 363         sf_list_dict can be a list of dict or a single dict
 364         """
 365         if isinstance(sf_list_dict, dict):
 366             sf_list_ = [sf_list_dict]
 367         elif isinstance(sf_list_dict, list):
 368             sf_list_ = sf_list_dict
 369         else:
 370             raise TypeError(
 371                 "param sf_list_dict must be a list or a dictionary")
 372         for sf in sf_list_:
 373             del sf['port_pair_group_parameters']
 374             sf['sfis'] = sf['port_pairs']
 375             del sf['port_pairs']
 376
 377     def __sfp_os2mano(self, sfp_list_dict):
 378         """Transform the openstack format (Port Chain) to mano format (SFP)
 379         sfp_list_dict can be a list of dict or a single dict
 380         """
 381         if isinstance(sfp_list_dict, dict):
 382             sfp_list_ = [sfp_list_dict]
 383         elif isinstance(sfp_list_dict, list):
 384             sfp_list_ = sfp_list_dict
 385         else:
 386             raise TypeError(
 387                 "param sfp_list_dict must be a list or a dictionary")
 388         for sfp in sfp_list_:
 389             params = sfp.pop('chain_parameters')
 390             sfc_encap = False
 391             if params:
 392                 correlation = params.get('correlation')
 393                 if correlation:
 394                     sfc_encap = True
 395             sfp['sfc_encap'] = sfc_encap
 396             sfp['spi'] = sfp.pop('chain_id')
 397             sfp['classifications'] = sfp.pop('flow_classifiers')
 398             sfp['service_functions'] = sfp.pop('port_pair_groups')
 399
 400     # placeholder for now; read TODO note below
 401     def _validate_classification(self, type, definition):
 402         # only legacy_flow_classifier Type is supported at this point
 403         return True
 404         # TODO(igordcard): this method should be an abstract method of an
 405         # abstract Classification class to be implemented by the specific
 406         # Types. Also, abstract vimconnector should call the validation
 407         # method before the implemented VIM connectors are called.
 408
 409     def _format_exception(self, exception):
 410         """Transform a keystone, nova, neutron  exception into a vimconn exception discovering the cause"""
 411
 412         message_error = str(exception)
 413         tip = ""
 414
 415         if isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound, ksExceptions.NotFound,
 416                                   gl1Exceptions.HTTPNotFound)):
 417             raise vimconn.VimConnNotFoundException(type(exception).__name__ + ": " + message_error)
 418         elif isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError,
 419                                     ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed)):
 420             if type(exception).__name__ == "SSLError":
 421                 tip = " (maybe option 'insecure' must be added to the VIM)"
 422             raise vimconn.VimConnConnectionException("Invalid URL or credentials{}: {}".format(tip, message_error))
 423         elif isinstance(exception, (KeyError, nvExceptions.BadRequest, ksExceptions.BadRequest)):
 424             raise vimconn.VimConnException(type(exception).__name__ + ": " + message_error)
 425         elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException,
 426                                     neExceptions.NeutronException)):
 427             raise vimconn.VimConnUnexpectedResponse(type(exception).__name__ + ": " + message_error)
 428         elif isinstance(exception, nvExceptions.Conflict):
 429             raise vimconn.VimConnConflictException(type(exception).__name__ + ": " + message_error)
 430         elif isinstance(exception, vimconn.VimConnException):
 431             raise exception
 432         else:  # ()
 433             self.logger.error("General Exception " + message_error, exc_info=True)
 434             raise vimconn.VimConnConnectionException(type(exception).__name__ + ": " + message_error)
 435
 436     def _get_ids_from_name(self):
 437         """
 438          Obtain ids from name of tenant and security_groups. Store at self .security_groups_id"
 439         :return: None
 440         """
 441         # get tenant_id if only tenant_name is supplied
 442         self._reload_connection()
 443         if not self.my_tenant_id:
 444             raise vimconn.VimConnConnectionException("Error getting tenant information from name={} id={}".
 445                                                      format(self.tenant_name, self.tenant_id))
 446         if self.config.get('security_groups') and not self.security_groups_id:
 447             # convert from name to id
 448             neutron_sg_list = self.neutron.list_security_groups(tenant_id=self.my_tenant_id)["security_groups"]
 449
 450             self.security_groups_id = []
 451             for sg in self.config.get('security_groups'):
 452                 for neutron_sg in neutron_sg_list:
 453                     if sg in (neutron_sg["id"], neutron_sg["name"]):
 454                         self.security_groups_id.append(neutron_sg["id"])
 455                         break
 456                 else:
 457                     self.security_groups_id = None
 458                     raise vimconn.VimConnConnectionException("Not found security group {} for this tenant".format(sg))
 459
 460     def check_vim_connectivity(self):
 461         # just get network list to check connectivity and credentials
 462         self.get_network_list(filter_dict={})
 463
 464     def get_tenant_list(self, filter_dict={}):
 465         """Obtain tenants of VIM
 466         filter_dict can contain the following keys:
 467             name: filter by tenant name
 468             id: filter by tenant uuid/id
 469             <other VIM specific>
 470         Returns the tenant list of dictionaries: [{'name':'<name>, 'id':'<id>, ...}, ...]
 471         """
 472         self.logger.debug("Getting tenants from VIM filter: '%s'", str(filter_dict))
 473         try:
 474             self._reload_connection()
 475             if self.api_version3:
 476                 project_class_list = self.keystone.projects.list(name=filter_dict.get("name"))
 477             else:
 478                 project_class_list = self.keystone.tenants.findall(**filter_dict)
 479             project_list = []
 480             for project in project_class_list:
 481                 if filter_dict.get('id') and filter_dict["id"] != project.id:
 482                     continue
 483                 project_list.append(project.to_dict())
 484             return project_list
 485         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError) as e:
 486             self._format_exception(e)
 487
 488     def new_tenant(self, tenant_name, tenant_description):
 489         """Adds a new tenant to openstack VIM. Returns the tenant identifier"""
 490         self.logger.debug("Adding a new tenant name: %s", tenant_name)
 491         try:
 492             self._reload_connection()
 493             if self.api_version3:
 494                 project = self.keystone.projects.create(tenant_name, self.config.get("project_domain_id", "default"),
 495                                                         description=tenant_description, is_domain=False)
 496             else:
 497                 project = self.keystone.tenants.create(tenant_name, tenant_description)
 498             return project.id
 499         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ksExceptions.BadRequest, ConnectionError)\
 500                 as e:
 501             self._format_exception(e)
 502
 503     def delete_tenant(self, tenant_id):
 504         """Delete a tenant from openstack VIM. Returns the old tenant identifier"""
 505         self.logger.debug("Deleting tenant %s from VIM", tenant_id)
 506         try:
 507             self._reload_connection()
 508             if self.api_version3:
 509                 self.keystone.projects.delete(tenant_id)
 510             else:
 511                 self.keystone.tenants.delete(tenant_id)
 512             return tenant_id
 513         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ksExceptions.NotFound, ConnectionError)\
 514                 as e:
 515             self._format_exception(e)
 516
 517     def new_network(self, net_name, net_type, ip_profile=None, shared=False, provider_network_profile=None):
 518         """Adds a tenant network to VIM
 519         Params:
 520             'net_name': name of the network
 521             'net_type': one of:
 522                 'bridge': overlay isolated network
 523                 'data':   underlay E-LAN network for Passthrough and SRIOV interfaces
 524                 'ptp':    underlay E-LINE network for Passthrough and SRIOV interfaces.
 525             'ip_profile': is a dict containing the IP parameters of the network
 526                 'ip_version': can be "IPv4" or "IPv6" (Currently only IPv4 is implemented)
 527                 'subnet_address': ip_prefix_schema, that is X.X.X.X/Y
 528                 'gateway_address': (Optional) ip_schema, that is X.X.X.X
 529                 'dns_address': (Optional) comma separated list of ip_schema, e.g. X.X.X.X[,X,X,X,X]
 530                 'dhcp_enabled': True or False
 531                 'dhcp_start_address': ip_schema, first IP to grant
 532                 'dhcp_count': number of IPs to grant.
 533             'shared': if this network can be seen/use by other tenants/organization
 534             'provider_network_profile': (optional) contains {segmentation-id: vlan, network-type: vlan|vxlan,
 535                                                              physical-network: physnet-label}
 536         Returns a tuple with the network identifier and created_items, or raises an exception on error
 537             created_items can be None or a dictionary where this method can include key-values that will be passed to
 538             the method delete_network. Can be used to store created segments, created l2gw connections, etc.
 539             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
 540             as not present.
 541         """
 542         self.logger.debug("Adding a new network to VIM name '%s', type '%s'", net_name, net_type)
 543         # self.logger.debug(">>>>>>>>>>>>>>>>>> IP profile %s", str(ip_profile))
 544
 545         try:
 546             vlan = None
 547             if provider_network_profile:
 548                 vlan = provider_network_profile.get("segmentation-id")
 549             new_net = None
 550             created_items = {}
 551             self._reload_connection()
 552             network_dict = {'name': net_name, 'admin_state_up': True}
 553             if net_type in ("data", "ptp"):
 554                 provider_physical_network = None
 555                 if provider_network_profile and provider_network_profile.get("physical-network"):
 556                     provider_physical_network = provider_network_profile.get("physical-network")
 557                     # provider-network must be one of the dataplane_physcial_netowrk if this is a list. If it is string
 558                     # or not declared, just ignore the checking
 559                     if isinstance(self.config.get('dataplane_physical_net'), (tuple, list)) and \
 560                             provider_physical_network not in self.config['dataplane_physical_net']:
 561                         raise vimconn.VimConnConflictException(
 562                             "Invalid parameter 'provider-network:physical-network' for network creation. '{}' is not "
 563                             "one of the declared list at VIM_config:dataplane_physical_net".format(
 564                                 provider_physical_network))
 565                 if not provider_physical_network:  # use the default dataplane_physical_net
 566                     provider_physical_network = self.config.get('dataplane_physical_net')
 567                     # if it is non empty list, use the first value. If it is a string use the value directly
 568                     if isinstance(provider_physical_network, (tuple, list)) and provider_physical_network:
 569                         provider_physical_network = provider_physical_network[0]
 570
 571                 if not provider_physical_network:
 572                     raise vimconn.VimConnConflictException(
 573                         "missing information needed for underlay networks. Provide 'dataplane_physical_net' "
 574                         "configuration at VIM or use the NS instantiation parameter 'provider-network.physical-network'"
 575                         " for the VLD")
 576
 577                 if not self.config.get('multisegment_support'):
 578                     network_dict["provider:physical_network"] = provider_physical_network
 579                     if provider_network_profile and "network-type" in provider_network_profile:
 580                         network_dict["provider:network_type"] = provider_network_profile["network-type"]
 581                     else:
 582                         network_dict["provider:network_type"] = self.config.get('dataplane_network_type', 'vlan')
 583                     if vlan:
 584                         network_dict["provider:segmentation_id"] = vlan
 585                 else:
 586                     # Multi-segment case
 587                     segment_list = []
 588                     segment1_dict = {
 589                         "provider:physical_network": '',
 590                         "provider:network_type": 'vxlan'
 591                     }
 592                     segment_list.append(segment1_dict)
 593                     segment2_dict = {
 594                         "provider:physical_network": provider_physical_network,
 595                         "provider:network_type": "vlan"
 596                     }
 597                     if vlan:
 598                         segment2_dict["provider:segmentation_id"] = vlan
 599                     elif self.config.get('multisegment_vlan_range'):
 600                         vlanID = self._generate_multisegment_vlanID()
 601                         segment2_dict["provider:segmentation_id"] = vlanID
 602                     # else
 603                     #     raise vimconn.VimConnConflictException(
 604                     #         "You must provide 'multisegment_vlan_range' at config dict before creating a multisegment
 605                     #         network")
 606                     segment_list.append(segment2_dict)
 607                     network_dict["segments"] = segment_list
 608
 609                 # VIO Specific Changes. It needs a concrete VLAN
 610                 if self.vim_type == "VIO" and vlan is None:
 611                     if self.config.get('dataplane_net_vlan_range') is None:
 612                         raise vimconn.VimConnConflictException(
 613                             "You must provide 'dataplane_net_vlan_range' in format [start_ID - end_ID] at VIM_config "
 614                             "for creating underlay networks")
 615                     network_dict["provider:segmentation_id"] = self._generate_vlanID()
 616
 617             network_dict["shared"] = shared
 618             if self.config.get("disable_network_port_security"):
 619                 network_dict["port_security_enabled"] = False
 620             new_net = self.neutron.create_network({'network': network_dict})
 621             # print new_net
 622             # create subnetwork, even if there is no profile
 623             if not ip_profile:
 624                 ip_profile = {}
 625             if not ip_profile.get('subnet_address'):
 626                 # Fake subnet is required
 627                 subnet_rand = random.randint(0, 255)
 628                 ip_profile['subnet_address'] = "192.168.{}.0/24".format(subnet_rand)
 629             if 'ip_version' not in ip_profile:
 630                 ip_profile['ip_version'] = "IPv4"
 631             subnet = {"name": net_name+"-subnet",
 632                       "network_id": new_net["network"]["id"],
 633                       "ip_version": 4 if ip_profile['ip_version'] == "IPv4" else 6,
 634                       "cidr": ip_profile['subnet_address']
 635                       }
 636             # Gateway should be set to None if not needed. Otherwise openstack assigns one by default
 637             if ip_profile.get('gateway_address'):
 638                 subnet['gateway_ip'] = ip_profile['gateway_address']
 639             else:
 640                 subnet['gateway_ip'] = None
 641             if ip_profile.get('dns_address'):
 642                 subnet['dns_nameservers'] = ip_profile['dns_address'].split(";")
 643             if 'dhcp_enabled' in ip_profile:
 644                 subnet['enable_dhcp'] = False if \
 645                     ip_profile['dhcp_enabled'] == "false" or ip_profile['dhcp_enabled'] is False else True
 646             if ip_profile.get('dhcp_start_address'):
 647                 subnet['allocation_pools'] = []
 648                 subnet['allocation_pools'].append(dict())
 649                 subnet['allocation_pools'][0]['start'] = ip_profile['dhcp_start_address']
 650             if ip_profile.get('dhcp_count'):
 651                 # parts = ip_profile['dhcp_start_address'].split('.')
 652                 # ip_int = (int(parts[0]) << 24) + (int(parts[1]) << 16) + (int(parts[2]) << 8) + int(parts[3])
 653                 ip_int = int(netaddr.IPAddress(ip_profile['dhcp_start_address']))
 654                 ip_int += ip_profile['dhcp_count'] - 1
 655                 ip_str = str(netaddr.IPAddress(ip_int))
 656                 subnet['allocation_pools'][0]['end'] = ip_str
 657             # self.logger.debug(">>>>>>>>>>>>>>>>>> Subnet: %s", str(subnet))
 658             self.neutron.create_subnet({"subnet": subnet})
 659
 660             if net_type == "data" and self.config.get('multisegment_support'):
 661                 if self.config.get('l2gw_support'):
 662                     l2gw_list = self.neutron.list_l2_gateways().get("l2_gateways", ())
 663                     for l2gw in l2gw_list:
 664                         l2gw_conn = {
 665                             "l2_gateway_id": l2gw["id"],
 666                             "network_id": new_net["network"]["id"],
 667                             "segmentation_id": str(vlanID),
 668                         }
 669                         new_l2gw_conn = self.neutron.create_l2_gateway_connection({"l2_gateway_connection": l2gw_conn})
 670                         created_items["l2gwconn:" + str(new_l2gw_conn["l2_gateway_connection"]["id"])] = True
 671             return new_net["network"]["id"], created_items
 672         except Exception as e:
 673             # delete l2gw connections (if any) before deleting the network
 674             for k, v in created_items.items():
 675                 if not v:  # skip already deleted
 676                     continue
 677                 try:
 678                     k_item, _, k_id = k.partition(":")
 679                     if k_item == "l2gwconn":
 680                         self.neutron.delete_l2_gateway_connection(k_id)
 681                 except Exception as e2:
 682                     self.logger.error("Error deleting l2 gateway connection: {}: {}".format(type(e2).__name__, e2))
 683             if new_net:
 684                 self.neutron.delete_network(new_net['network']['id'])
 685             self._format_exception(e)
 686
 687     def get_network_list(self, filter_dict={}):
 688         """Obtain tenant networks of VIM
 689         Filter_dict can be:
 690             name: network name
 691             id: network uuid
 692             shared: boolean
 693             tenant_id: tenant
 694             admin_state_up: boolean
 695             status: 'ACTIVE'
 696         Returns the network list of dictionaries
 697         """
 698         self.logger.debug("Getting network from VIM filter: '%s'", str(filter_dict))
 699         try:
 700             self._reload_connection()
 701             filter_dict_os = filter_dict.copy()
 702             if self.api_version3 and "tenant_id" in filter_dict_os:
 703                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')  # TODO check
 704             net_dict = self.neutron.list_networks(**filter_dict_os)
 705             net_list = net_dict["networks"]
 706             self.__net_os2mano(net_list)
 707             return net_list
 708         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException,
 709                 ConnectionError) as e:
 710             self._format_exception(e)
 711
 712     def get_network(self, net_id):
 713         """Obtain details of network from VIM
 714         Returns the network information from a network id"""
 715         self.logger.debug(" Getting tenant network %s from VIM", net_id)
 716         filter_dict = {"id": net_id}
 717         net_list = self.get_network_list(filter_dict)
 718         if len(net_list) == 0:
 719             raise vimconn.VimConnNotFoundException("Network '{}' not found".format(net_id))
 720         elif len(net_list) > 1:
 721             raise vimconn.VimConnConflictException("Found more than one network with this criteria")
 722         net = net_list[0]
 723         subnets = []
 724         for subnet_id in net.get("subnets", ()):
 725             try:
 726                 subnet = self.neutron.show_subnet(subnet_id)
 727             except Exception as e:
 728                 self.logger.error("osconnector.get_network(): Error getting subnet %s %s" % (net_id, str(e)))
 729                 subnet = {"id": subnet_id, "fault": str(e)}
 730             subnets.append(subnet)
 731         net["subnets"] = subnets
 732         net["encapsulation"] = net.get('provider:network_type')
 733         net["encapsulation_type"] = net.get('provider:network_type')
 734         net["segmentation_id"] = net.get('provider:segmentation_id')
 735         net["encapsulation_id"] = net.get('provider:segmentation_id')
 736         return net
 737
 738     def delete_network(self, net_id, created_items=None):
 739         """
 740         Removes a tenant network from VIM and its associated elements
 741         :param net_id: VIM identifier of the network, provided by method new_network
 742         :param created_items: dictionary with extra items to be deleted. provided by method new_network
 743         Returns the network identifier or raises an exception upon error or when network is not found
 744         """
 745         self.logger.debug("Deleting network '%s' from VIM", net_id)
 746         if created_items is None:
 747             created_items = {}
 748         try:
 749             self._reload_connection()
 750             # delete l2gw connections (if any) before deleting the network
 751             for k, v in created_items.items():
 752                 if not v:  # skip already deleted
 753                     continue
 754                 try:
 755                     k_item, _, k_id = k.partition(":")
 756                     if k_item == "l2gwconn":
 757                         self.neutron.delete_l2_gateway_connection(k_id)
 758                 except Exception as e:
 759                     self.logger.error("Error deleting l2 gateway connection: {}: {}".format(type(e).__name__, e))
 760             # delete VM ports attached to this networks before the network
 761             ports = self.neutron.list_ports(network_id=net_id)
 762             for p in ports['ports']:
 763                 try:
 764                     self.neutron.delete_port(p["id"])
 765                 except Exception as e:
 766                     self.logger.error("Error deleting port %s: %s", p["id"], str(e))
 767             self.neutron.delete_network(net_id)
 768             return net_id
 769         except (neExceptions.ConnectionFailed, neExceptions.NetworkNotFoundClient, neExceptions.NeutronException,
 770                 ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 771             self._format_exception(e)
 772
 773     def refresh_nets_status(self, net_list):
 774         """Get the status of the networks
 775            Params: the list of network identifiers
 776            Returns a dictionary with:
 777                 net_id:         #VIM id of this network
 778                     status:     #Mandatory. Text with one of:
 779                                 #  DELETED (not found at vim)
 780                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 781                                 #  OTHER (Vim reported other status not understood)
 782                                 #  ERROR (VIM indicates an ERROR status)
 783                                 #  ACTIVE, INACTIVE, DOWN (admin down),
 784                                 #  BUILD (on building process)
 785                                 #
 786                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 787                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 788
 789         """
 790         net_dict = {}
 791         for net_id in net_list:
 792             net = {}
 793             try:
 794                 net_vim = self.get_network(net_id)
 795                 if net_vim['status'] in netStatus2manoFormat:
 796                     net["status"] = netStatus2manoFormat[net_vim['status']]
 797                 else:
 798                     net["status"] = "OTHER"
 799                     net["error_msg"] = "VIM status reported " + net_vim['status']
 800
 801                 if net['status'] == "ACTIVE" and not net_vim['admin_state_up']:
 802                     net['status'] = 'DOWN'
 803
 804                 net['vim_info'] = self.serialize(net_vim)
 805
 806                 if net_vim.get('fault'):  # TODO
 807                     net['error_msg'] = str(net_vim['fault'])
 808             except vimconn.VimConnNotFoundException as e:
 809                 self.logger.error("Exception getting net status: %s", str(e))
 810                 net['status'] = "DELETED"
 811                 net['error_msg'] = str(e)
 812             except vimconn.VimConnException as e:
 813                 self.logger.error("Exception getting net status: %s", str(e))
 814                 net['status'] = "VIM_ERROR"
 815                 net['error_msg'] = str(e)
 816             net_dict[net_id] = net
 817         return net_dict
 818
 819     def get_flavor(self, flavor_id):
 820         """Obtain flavor details from the  VIM. Returns the flavor dict details"""
 821         self.logger.debug("Getting flavor '%s'", flavor_id)
 822         try:
 823             self._reload_connection()
 824             flavor = self.nova.flavors.find(id=flavor_id)
 825             # TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 826             return flavor.to_dict()
 827         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException,
 828                 ConnectionError) as e:
 829             self._format_exception(e)
 830
 831     def get_flavor_id_from_data(self, flavor_dict):
 832         """Obtain flavor id that match the flavor description
 833            Returns the flavor_id or raises a vimconnNotFoundException
 834            flavor_dict: contains the required ram, vcpus, disk
 835            If 'use_existing_flavors' is set to True at config, the closer flavor that provides same or more ram, vcpus
 836                 and disk is returned. Otherwise a flavor with exactly same ram, vcpus and disk is returned or a
 837                 vimconnNotFoundException is raised
 838         """
 839         exact_match = False if self.config.get('use_existing_flavors') else True
 840         try:
 841             self._reload_connection()
 842             flavor_candidate_id = None
 843             flavor_candidate_data = (10000, 10000, 10000)
 844             flavor_target = (flavor_dict["ram"], flavor_dict["vcpus"], flavor_dict["disk"])
 845             # numa=None
 846             extended = flavor_dict.get("extended", {})
 847             if extended:
 848                 # TODO
 849                 raise vimconn.VimConnNotFoundException("Flavor with EPA still not implemented")
 850                 # if len(numas) > 1:
 851                 #     raise vimconn.VimConnNotFoundException("Cannot find any flavor with more than one numa")
 852                 # numa=numas[0]
 853                 # numas = extended.get("numas")
 854             for flavor in self.nova.flavors.list():
 855                 epa = flavor.get_keys()
 856                 if epa:
 857                     continue
 858                     # TODO
 859                 flavor_data = (flavor.ram, flavor.vcpus, flavor.disk)
 860                 if flavor_data == flavor_target:
 861                     return flavor.id
 862                 elif not exact_match and flavor_target < flavor_data < flavor_candidate_data:
 863                     flavor_candidate_id = flavor.id
 864                     flavor_candidate_data = flavor_data
 865             if not exact_match and flavor_candidate_id:
 866                 return flavor_candidate_id
 867             raise vimconn.VimConnNotFoundException("Cannot find any flavor matching '{}'".format(flavor_dict))
 868         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException,
 869                 ConnectionError) as e:
 870             self._format_exception(e)
 871
 872     def process_resource_quota(self, quota, prefix, extra_specs):
 873         """
 874         :param prefix:
 875         :param extra_specs:
 876         :return:
 877         """
 878         if 'limit' in quota:
 879             extra_specs["quota:" + prefix + "_limit"] = quota['limit']
 880         if 'reserve' in quota:
 881             extra_specs["quota:" + prefix + "_reservation"] = quota['reserve']
 882         if 'shares' in quota:
 883             extra_specs["quota:" + prefix + "_shares_level"] = "custom"
 884             extra_specs["quota:" + prefix + "_shares_share"] = quota['shares']
 885
 886     def new_flavor(self, flavor_data, change_name_if_used=True):
 887         """Adds a tenant flavor to openstack VIM
 888         if change_name_if_used is True, it will change name in case of conflict, because it is not supported name
 889          repetition
 890         Returns the flavor identifier
 891         """
 892         self.logger.debug("Adding flavor '%s'", str(flavor_data))
 893         retry = 0
 894         max_retries = 3
 895         name_suffix = 0
 896         try:
 897             name = flavor_data['name']
 898             while retry < max_retries:
 899                 retry += 1
 900                 try:
 901                     self._reload_connection()
 902                     if change_name_if_used:
 903                         # get used names
 904                         fl_names = []
 905                         fl = self.nova.flavors.list()
 906                         for f in fl:
 907                             fl_names.append(f.name)
 908                         while name in fl_names:
 909                             name_suffix += 1
 910                             name = flavor_data['name']+"-" + str(name_suffix)
 911
 912                     ram = flavor_data.get('ram', 64)
 913                     vcpus = flavor_data.get('vcpus', 1)
 914                     extra_specs = {}
 915
 916                     extended = flavor_data.get("extended")
 917                     if extended:
 918                         numas = extended.get("numas")
 919                         if numas:
 920                             numa_nodes = len(numas)
 921                             if numa_nodes > 1:
 922                                 return -1, "Can not add flavor with more than one numa"
 923                             extra_specs["hw:numa_nodes"] = str(numa_nodes)
 924                             extra_specs["hw:mem_page_size"] = "large"
 925                             extra_specs["hw:cpu_policy"] = "dedicated"
 926                             extra_specs["hw:numa_mempolicy"] = "strict"
 927                             if self.vim_type == "VIO":
 928                                 extra_specs["vmware:extra_config"] = '{"numa.nodeAffinity":"0"}'
 929                                 extra_specs["vmware:latency_sensitivity_level"] = "high"
 930                             for numa in numas:
 931                                 # overwrite ram and vcpus
 932                                 # check if key 'memory' is present in numa else use ram value at flavor
 933                                 if 'memory' in numa:
 934                                     ram = numa['memory']*1024
 935                                 # See for reference: https://specs.openstack.org/openstack/nova-specs/specs/mitaka/
 936                                 # implemented/virt-driver-cpu-thread-pinning.html
 937                                 extra_specs["hw:cpu_sockets"] = 1
 938                                 if 'paired-threads' in numa:
 939                                     vcpus = numa['paired-threads']*2
 940                                     # cpu_thread_policy "require" implies that the compute node must have an
 941                                     # STM architecture
 942                                     extra_specs["hw:cpu_thread_policy"] = "require"
 943                                     extra_specs["hw:cpu_policy"] = "dedicated"
 944                                 elif 'cores' in numa:
 945                                     vcpus = numa['cores']
 946                                     # cpu_thread_policy "prefer" implies that the host must not have an SMT
 947                                     # architecture, or a non-SMT architecture will be emulated
 948                                     extra_specs["hw:cpu_thread_policy"] = "isolate"
 949                                     extra_specs["hw:cpu_policy"] = "dedicated"
 950                                 elif 'threads' in numa:
 951                                     vcpus = numa['threads']
 952                                     # cpu_thread_policy "prefer" implies that the host may or may not have an SMT
 953                                     # architecture
 954                                     extra_specs["hw:cpu_thread_policy"] = "prefer"
 955                                     extra_specs["hw:cpu_policy"] = "dedicated"
 956                                 # for interface in numa.get("interfaces",() ):
 957                                 #     if interface["dedicated"]=="yes":
 958                                 #         raise vimconn.VimConnException("Passthrough interfaces are not supported
 959                                 #         for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable)
 960                                 #     #TODO, add the key 'pci_passthrough:alias"="<label at config>:<number ifaces>"'
 961                                 #      when a way to connect it is available
 962                         elif extended.get("cpu-quota"):
 963                             self.process_resource_quota(extended.get("cpu-quota"), "cpu", extra_specs)
 964                         if extended.get("mem-quota"):
 965                             self.process_resource_quota(extended.get("mem-quota"), "memory", extra_specs)
 966                         if extended.get("vif-quota"):
 967                             self.process_resource_quota(extended.get("vif-quota"), "vif", extra_specs)
 968                         if extended.get("disk-io-quota"):
 969                             self.process_resource_quota(extended.get("disk-io-quota"), "disk_io", extra_specs)
 970                     # create flavor
 971                     new_flavor = self.nova.flavors.create(name,
 972                                                           ram,
 973                                                           vcpus,
 974                                                           flavor_data.get('disk', 0),
 975                                                           is_public=flavor_data.get('is_public', True)
 976                                                           )
 977                     # add metadata
 978                     if extra_specs:
 979                         new_flavor.set_keys(extra_specs)
 980                     return new_flavor.id
 981                 except nvExceptions.Conflict as e:
 982                     if change_name_if_used and retry < max_retries:
 983                         continue
 984                     self._format_exception(e)
 985         # except nvExceptions.BadRequest as e:
 986         except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError, KeyError) as e:
 987             self._format_exception(e)
 988
 989     def delete_flavor(self, flavor_id):
 990         """Deletes a tenant flavor from openstack VIM. Returns the old flavor_id
 991         """
 992         try:
 993             self._reload_connection()
 994             self.nova.flavors.delete(flavor_id)
 995             return flavor_id
 996         # except nvExceptions.BadRequest as e:
 997         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException,
 998                 ConnectionError) as e:
 999             self._format_exception(e)
1000
1001     def new_image(self, image_dict):
1002         """
1003         Adds a tenant image to VIM. imge_dict is a dictionary with:
1004             name: name
1005             disk_format: qcow2, vhd, vmdk, raw (by default), ...
1006             location: path or URI
1007             public: "yes" or "no"
1008             metadata: metadata of the image
1009         Returns the image_id
1010         """
1011         retry = 0
1012         max_retries = 3
1013         while retry < max_retries:
1014             retry += 1
1015             try:
1016                 self._reload_connection()
1017                 # determine format  http://docs.openstack.org/developer/glance/formats.html
1018                 if "disk_format" in image_dict:
1019                     disk_format = image_dict["disk_format"]
1020                 else:  # autodiscover based on extension
1021                     if image_dict['location'].endswith(".qcow2"):
1022                         disk_format = "qcow2"
1023                     elif image_dict['location'].endswith(".vhd"):
1024                         disk_format = "vhd"
1025                     elif image_dict['location'].endswith(".vmdk"):
1026                         disk_format = "vmdk"
1027                     elif image_dict['location'].endswith(".vdi"):
1028                         disk_format = "vdi"
1029                     elif image_dict['location'].endswith(".iso"):
1030                         disk_format = "iso"
1031                     elif image_dict['location'].endswith(".aki"):
1032                         disk_format = "aki"
1033                     elif image_dict['location'].endswith(".ari"):
1034                         disk_format = "ari"
1035                     elif image_dict['location'].endswith(".ami"):
1036                         disk_format = "ami"
1037                     else:
1038                         disk_format = "raw"
1039                 self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location'])
1040                 if self.vim_type == "VIO":
1041                     container_format = "bare"
1042                     if 'container_format' in image_dict:
1043                         container_format = image_dict['container_format']
1044                     new_image = self.glance.images.create(name=image_dict['name'], container_format=container_format,
1045                                                           disk_format=disk_format)
1046                 else:
1047                     new_image = self.glance.images.create(name=image_dict['name'])
1048                 if image_dict['location'].startswith("http"):
1049                     # TODO there is not a method to direct download. It must be downloaded locally with requests
1050                     raise vimconn.VimConnNotImplemented("Cannot create image from URL")
1051                 else:  # local path
1052                     with open(image_dict['location']) as fimage:
1053                         self.glance.images.upload(new_image.id, fimage)
1054                         # new_image = self.glancev1.images.create(name=image_dict['name'], is_public=
1055                         #  image_dict.get('public',"yes")=="yes",
1056                         #    container_format="bare", data=fimage, disk_format=disk_format)
1057                 metadata_to_load = image_dict.get('metadata')
1058                 # TODO location is a reserved word for current openstack versions. fixed for VIO please check
1059                 #  for openstack
1060                 if self.vim_type == "VIO":
1061                     metadata_to_load['upload_location'] = image_dict['location']
1062                 else:
1063                     metadata_to_load['location'] = image_dict['location']
1064                 self.glance.images.update(new_image.id, **metadata_to_load)
1065                 return new_image.id
1066             except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e:
1067                 self._format_exception(e)
1068             except (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError, ConnectionError) as e:
1069                 if retry == max_retries:
1070                     continue
1071                 self._format_exception(e)
1072             except IOError as e:  # can not open the file
1073                 raise vimconn.VimConnConnectionException("{}: {} for {}".format(type(e).__name__, e,
1074                                                                                 image_dict['location']),
1075                                                          http_code=vimconn.HTTP_Bad_Request)
1076
1077     def delete_image(self, image_id):
1078         """Deletes a tenant image from openstack VIM. Returns the old id
1079         """
1080         try:
1081             self._reload_connection()
1082             self.glance.images.delete(image_id)
1083             return image_id
1084         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException,
1085                 gl1Exceptions.CommunicationError, gl1Exceptions.HTTPNotFound, ConnectionError) as e:  # TODO remove
1086             self._format_exception(e)
1087
1088     def get_image_id_from_path(self, path):
1089         """Get the image id from image path in the VIM database. Returns the image_id"""
1090         try:
1091             self._reload_connection()
1092             images = self.glance.images.list()
1093             for image in images:
1094                 if image.metadata.get("location") == path:
1095                     return image.id
1096             raise vimconn.VimConnNotFoundException("image with location '{}' not found".format(path))
1097         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError,
1098                 ConnectionError) as e:
1099             self._format_exception(e)
1100
1101     def get_image_list(self, filter_dict={}):
1102         """Obtain tenant images from VIM
1103         Filter_dict can be:
1104             id: image id
1105             name: image name
1106             checksum: image checksum
1107         Returns the image list of dictionaries:
1108             [{<the fields at Filter_dict plus some VIM specific>}, ...]
1109             List can be empty
1110         """
1111         self.logger.debug("Getting image list from VIM filter: '%s'", str(filter_dict))
1112         try:
1113             self._reload_connection()
1114             # filter_dict_os = filter_dict.copy()
1115             # First we filter by the available filter fields: name, id. The others are removed.
1116             image_list = self.glance.images.list()
1117             filtered_list = []
1118             for image in image_list:
1119                 try:
1120                     if filter_dict.get("name") and image["name"] != filter_dict["name"]:
1121                         continue
1122                     if filter_dict.get("id") and image["id"] != filter_dict["id"]:
1123                         continue
1124                     if filter_dict.get("checksum") and image["checksum"] != filter_dict["checksum"]:
1125                         continue
1126
1127                     filtered_list.append(image.copy())
1128                 except gl1Exceptions.HTTPNotFound:
1129                     pass
1130             return filtered_list
1131         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError,
1132                 ConnectionError) as e:
1133             self._format_exception(e)
1134
1135     def __wait_for_vm(self, vm_id, status):
1136         """wait until vm is in the desired status and return True.
1137         If the VM gets in ERROR status, return false.
1138         If the timeout is reached generate an exception"""
1139         elapsed_time = 0
1140         while elapsed_time < server_timeout:
1141             vm_status = self.nova.servers.get(vm_id).status
1142             if vm_status == status:
1143                 return True
1144             if vm_status == 'ERROR':
1145                 return False
1146             time.sleep(5)
1147             elapsed_time += 5
1148
1149         # if we exceeded the timeout rollback
1150         if elapsed_time >= server_timeout:
1151             raise vimconn.VimConnException('Timeout waiting for instance ' + vm_id + ' to get ' + status,
1152                                            http_code=vimconn.HTTP_Request_Timeout)
1153
1154     def _get_openstack_availablity_zones(self):
1155         """
1156         Get from openstack availability zones available
1157         :return:
1158         """
1159         try:
1160             openstack_availability_zone = self.nova.availability_zones.list()
1161             openstack_availability_zone = [str(zone.zoneName) for zone in openstack_availability_zone
1162                                            if zone.zoneName != 'internal']
1163             return openstack_availability_zone
1164         except Exception:
1165             return None
1166
1167     def _set_availablity_zones(self):
1168         """
1169         Set vim availablity zone
1170         :return:
1171         """
1172
1173         if 'availability_zone' in self.config:
1174             vim_availability_zones = self.config.get('availability_zone')
1175             if isinstance(vim_availability_zones, str):
1176                 self.availability_zone = [vim_availability_zones]
1177             elif isinstance(vim_availability_zones, list):
1178                 self.availability_zone = vim_availability_zones
1179         else:
1180             self.availability_zone = self._get_openstack_availablity_zones()
1181
1182     def _get_vm_availability_zone(self, availability_zone_index, availability_zone_list):
1183         """
1184         Return thge availability zone to be used by the created VM.
1185         :return: The VIM availability zone to be used or None
1186         """
1187         if availability_zone_index is None:
1188             if not self.config.get('availability_zone'):
1189                 return None
1190             elif isinstance(self.config.get('availability_zone'), str):
1191                 return self.config['availability_zone']
1192             else:
1193                 # TODO consider using a different parameter at config for default AV and AV list match
1194                 return self.config['availability_zone'][0]
1195
1196         vim_availability_zones = self.availability_zone
1197         # check if VIM offer enough availability zones describe in the VNFD
1198         if vim_availability_zones and len(availability_zone_list) <= len(vim_availability_zones):
1199             # check if all the names of NFV AV match VIM AV names
1200             match_by_index = False
1201             for av in availability_zone_list:
1202                 if av not in vim_availability_zones:
1203                     match_by_index = True
1204                     break
1205             if match_by_index:
1206                 return vim_availability_zones[availability_zone_index]
1207             else:
1208                 return availability_zone_list[availability_zone_index]
1209         else:
1210             raise vimconn.VimConnConflictException("No enough availability zones at VIM for this deployment")
1211
1212     def new_vminstance(self, name, description, start, image_id, flavor_id, net_list, cloud_config=None, disk_list=None,
1213                        availability_zone_index=None, availability_zone_list=None):
1214         """Adds a VM instance to VIM
1215         Params:
1216             start: indicates if VM must start or boot in pause mode. Ignored
1217             image_id,flavor_id: iamge and flavor uuid
1218             net_list: list of interfaces, each one is a dictionary with:
1219                 name:
1220                 net_id: network uuid to connect
1221                 vpci: virtual vcpi to assign, ignored because openstack lack #TODO
1222                 model: interface model, ignored #TODO
1223                 mac_address: used for  SR-IOV ifaces #TODO for other types
1224                 use: 'data', 'bridge',  'mgmt'
1225                 type: 'virtual', 'PCI-PASSTHROUGH'('PF'), 'SR-IOV'('VF'), 'VFnotShared'
1226                 vim_id: filled/added by this function
1227                 floating_ip: True/False (or it can be None)
1228                 port_security: True/False
1229             'cloud_config': (optional) dictionary with:
1230                 'key-pairs': (optional) list of strings with the public key to be inserted to the default user
1231                 'users': (optional) list of users to be inserted, each item is a dict with:
1232                     'name': (mandatory) user name,
1233                     'key-pairs': (optional) list of strings with the public key to be inserted to the user
1234                 'user-data': (optional) string is a text script to be passed directly to cloud-init
1235                 'config-files': (optional). List of files to be transferred. Each item is a dict with:
1236                     'dest': (mandatory) string with the destination absolute path
1237                     'encoding': (optional, by default text). Can be one of:
1238                         'b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64'
1239                     'content' (mandatory): string with the content of the file
1240                     'permissions': (optional) string with file permissions, typically octal notation '0644'
1241                     'owner': (optional) file owner, string with the format 'owner:group'
1242                 'boot-data-drive': boolean to indicate if user-data must be passed using a boot drive (hard disk)
1243             'disk_list': (optional) list with additional disks to the VM. Each item is a dict with:
1244                 'image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted
1245                 'size': (mandatory) string with the size of the disk in GB
1246                 'vim_id' (optional) should use this existing volume id
1247             availability_zone_index: Index of availability_zone_list to use for this this VM. None if not AV required
1248             availability_zone_list: list of availability zones given by user in the VNFD descriptor.  Ignore if
1249                 availability_zone_index is None
1250                 #TODO ip, security groups
1251         Returns a tuple with the instance identifier and created_items or raises an exception on error
1252             created_items can be None or a dictionary where this method can include key-values that will be passed to
1253             the method delete_vminstance and action_vminstance. Can be used to store created ports, volumes, etc.
1254             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
1255             as not present.
1256         """
1257         self.logger.debug("new_vminstance input: image='%s' flavor='%s' nics='%s'", image_id, flavor_id, str(net_list))
1258         try:
1259             server = None
1260             created_items = {}
1261             # metadata = {}
1262             net_list_vim = []
1263             external_network = []
1264             # ^list of external networks to be connected to instance, later on used to create floating_ip
1265             no_secured_ports = []   # List of port-is with port-security disabled
1266             self._reload_connection()
1267             # metadata_vpci = {}   # For a specific neutron plugin
1268             block_device_mapping = None
1269
1270             for net in net_list:
1271                 if not net.get("net_id"):   # skip non connected iface
1272                     continue
1273
1274                 port_dict = {
1275                     "network_id": net["net_id"],
1276                     "name": net.get("name"),
1277                     "admin_state_up": True
1278                 }
1279                 if self.config.get("security_groups") and net.get("port_security") is not False and \
1280                         not self.config.get("no_port_security_extension"):
1281                     if not self.security_groups_id:
1282                         self._get_ids_from_name()
1283                     port_dict["security_groups"] = self.security_groups_id
1284
1285                 if net["type"] == "virtual":
1286                     pass
1287                     # if "vpci" in net:
1288                     #     metadata_vpci[ net["net_id"] ] = [[ net["vpci"], "" ]]
1289                 elif net["type"] == "VF" or net["type"] == "SR-IOV":  # for VF
1290                     # if "vpci" in net:
1291                     #     if "VF" not in metadata_vpci:
1292                     #         metadata_vpci["VF"]=[]
1293                     #     metadata_vpci["VF"].append([ net["vpci"], "" ])
1294                     port_dict["binding:vnic_type"] = "direct"
1295                     # VIO specific Changes
1296                     if self.vim_type == "VIO":
1297                         # Need to create port with port_security_enabled = False and no-security-groups
1298                         port_dict["port_security_enabled"] = False
1299                         port_dict["provider_security_groups"] = []
1300                         port_dict["security_groups"] = []
1301                 else:   # For PT PCI-PASSTHROUGH
1302                     # if "vpci" in net:
1303                     #     if "PF" not in metadata_vpci:
1304                     #         metadata_vpci["PF"]=[]
1305                     #     metadata_vpci["PF"].append([ net["vpci"], "" ])
1306                     port_dict["binding:vnic_type"] = "direct-physical"
1307                 if not port_dict["name"]:
1308                     port_dict["name"] = name
1309                 if net.get("mac_address"):
1310                     port_dict["mac_address"] = net["mac_address"]
1311                 if net.get("ip_address"):
1312                     port_dict["fixed_ips"] = [{'ip_address': net["ip_address"]}]
1313                     # TODO add 'subnet_id': <subnet_id>
1314                 new_port = self.neutron.create_port({"port": port_dict})
1315                 created_items["port:" + str(new_port["port"]["id"])] = True
1316                 net["mac_adress"] = new_port["port"]["mac_address"]
1317                 net["vim_id"] = new_port["port"]["id"]
1318                 # if try to use a network without subnetwork, it will return a emtpy list
1319                 fixed_ips = new_port["port"].get("fixed_ips")
1320                 if fixed_ips:
1321                     net["ip"] = fixed_ips[0].get("ip_address")
1322                 else:
1323                     net["ip"] = None
1324
1325                 port = {"port-id": new_port["port"]["id"]}
1326                 if float(self.nova.api_version.get_string()) >= 2.32:
1327                     port["tag"] = new_port["port"]["name"]
1328                 net_list_vim.append(port)
1329
1330                 if net.get('floating_ip', False):
1331                     net['exit_on_floating_ip_error'] = True
1332                     external_network.append(net)
1333                 elif net['use'] == 'mgmt' and self.config.get('use_floating_ip'):
1334                     net['exit_on_floating_ip_error'] = False
1335                     external_network.append(net)
1336                     net['floating_ip'] = self.config.get('use_floating_ip')
1337
1338                 # If port security is disabled when the port has not yet been attached to the VM, then all vm traffic
1339                 # is dropped.
1340                 # As a workaround we wait until the VM is active and then disable the port-security
1341                 if net.get("port_security") is False and not self.config.get("no_port_security_extension"):
1342                     no_secured_ports.append((new_port["port"]["id"], net.get("port_security_disable_strategy")))
1343
1344             # if metadata_vpci:
1345             #     metadata = {"pci_assignement": json.dumps(metadata_vpci)}
1346             #     if len(metadata["pci_assignement"]) >255:
1347             #         #limit the metadata size
1348             #         #metadata["pci_assignement"] = metadata["pci_assignement"][0:255]
1349             #         self.logger.warn("Metadata deleted since it exceeds the expected length (255) ")
1350             #         metadata = {}
1351
1352             self.logger.debug("name '%s' image_id '%s'flavor_id '%s' net_list_vim '%s' description '%s'",
1353                               name, image_id, flavor_id, str(net_list_vim), description)
1354
1355             # cloud config
1356             config_drive, userdata = self._create_user_data(cloud_config)
1357
1358             # Create additional volumes in case these are present in disk_list
1359             base_disk_index = ord('b')
1360             if disk_list:
1361                 block_device_mapping = {}
1362                 for disk in disk_list:
1363                     if disk.get('vim_id'):
1364                         block_device_mapping['_vd' + chr(base_disk_index)] = disk['vim_id']
1365                     else:
1366                         if 'image_id' in disk:
1367                             volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1368                                                                 chr(base_disk_index), imageRef=disk['image_id'])
1369                         else:
1370                             volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1371                                                                 chr(base_disk_index))
1372                         created_items["volume:" + str(volume.id)] = True
1373                         block_device_mapping['_vd' + chr(base_disk_index)] = volume.id
1374                     base_disk_index += 1
1375
1376                 # Wait until created volumes are with status available
1377                 elapsed_time = 0
1378                 while elapsed_time < volume_timeout:
1379                     for created_item in created_items:
1380                         v, _, volume_id = created_item.partition(":")
1381                         if v == 'volume':
1382                             if self.cinder.volumes.get(volume_id).status != 'available':
1383                                 break
1384                     else:  # all ready: break from while
1385                         break
1386                     time.sleep(5)
1387                     elapsed_time += 5
1388                 # If we exceeded the timeout rollback
1389                 if elapsed_time >= volume_timeout:
1390                     raise vimconn.VimConnException('Timeout creating volumes for instance ' + name,
1391                                                    http_code=vimconn.HTTP_Request_Timeout)
1392             # get availability Zone
1393             vm_av_zone = self._get_vm_availability_zone(availability_zone_index, availability_zone_list)
1394
1395             self.logger.debug("nova.servers.create({}, {}, {}, nics={}, security_groups={}, "
1396                               "availability_zone={}, key_name={}, userdata={}, config_drive={}, "
1397                               "block_device_mapping={})".format(name, image_id, flavor_id, net_list_vim,
1398                                                                 self.config.get("security_groups"), vm_av_zone,
1399                                                                 self.config.get('keypair'), userdata, config_drive,
1400                                                                 block_device_mapping))
1401             server = self.nova.servers.create(name, image_id, flavor_id, nics=net_list_vim,
1402                                               security_groups=self.config.get("security_groups"),
1403                                               # TODO remove security_groups in future versions. Already at neutron port
1404                                               availability_zone=vm_av_zone,
1405                                               key_name=self.config.get('keypair'),
1406                                               userdata=userdata,
1407                                               config_drive=config_drive,
1408                                               block_device_mapping=block_device_mapping
1409                                               )  # , description=description)
1410
1411             vm_start_time = time.time()
1412             # Previously mentioned workaround to wait until the VM is active and then disable the port-security
1413             if no_secured_ports:
1414                 self.__wait_for_vm(server.id, 'ACTIVE')
1415
1416             for port in no_secured_ports:
1417                 port_update = {
1418                     "port": {
1419                         "port_security_enabled": False,
1420                         "security_groups": None
1421                     }
1422                 }
1423
1424                 if port[1] == "allow-address-pairs":
1425                     port_update = {
1426                         "port": {
1427                             "allowed_address_pairs": [
1428                                 {
1429                                     "ip_address": "0.0.0.0/0"
1430                                 }
1431                             ]
1432                         }
1433                     }
1434
1435                 try:
1436                     self.neutron.update_port(port[0], port_update)
1437                 except Exception:
1438                     raise vimconn.VimConnException(
1439                         "It was not possible to disable port security for port {}"
1440                         .format(port[0])
1441                     )
1442
1443             # print "DONE :-)", server
1444
1445             # pool_id = None
1446             for floating_network in external_network:
1447                 try:
1448                     assigned = False
1449                     floating_ip_retries = 3
1450                     # In case of RO in HA there can be conflicts, two RO trying to assign same floating IP, so retry
1451                     # several times
1452                     while not assigned:
1453                         floating_ips = self.neutron.list_floatingips().get("floatingips", ())
1454                         random.shuffle(floating_ips)   # randomize
1455                         for fip in floating_ips:
1456                             if fip.get("port_id") or fip.get('tenant_id') != server.tenant_id:
1457                                 continue
1458                             if isinstance(floating_network['floating_ip'], str):
1459                                 if fip.get("floating_network_id") != floating_network['floating_ip']:
1460                                     continue
1461                             free_floating_ip = fip["id"]
1462                             break
1463                         else:
1464                             if isinstance(floating_network['floating_ip'], str) and \
1465                                     floating_network['floating_ip'].lower() != "true":
1466                                 pool_id = floating_network['floating_ip']
1467                             else:
1468                                 # Find the external network
1469                                 external_nets = list()
1470                                 for net in self.neutron.list_networks()['networks']:
1471                                     if net['router:external']:
1472                                         external_nets.append(net)
1473
1474                                 if len(external_nets) == 0:
1475                                     raise vimconn.VimConnException(
1476                                         "Cannot create floating_ip automatically since no external network is present",
1477                                         http_code=vimconn.HTTP_Conflict)
1478                                 if len(external_nets) > 1:
1479                                     raise vimconn.VimConnException(
1480                                         "Cannot create floating_ip automatically since multiple external networks are"
1481                                         " present", http_code=vimconn.HTTP_Conflict)
1482
1483                                 pool_id = external_nets[0].get('id')
1484                             param = {'floatingip': {'floating_network_id': pool_id, 'tenant_id': server.tenant_id}}
1485                             try:
1486                                 # self.logger.debug("Creating floating IP")
1487                                 new_floating_ip = self.neutron.create_floatingip(param)
1488                                 free_floating_ip = new_floating_ip['floatingip']['id']
1489                                 created_items["floating_ip:" + str(free_floating_ip)] = True
1490                             except Exception as e:
1491                                 raise vimconn.VimConnException(type(e).__name__ + ": Cannot create new floating_ip " +
1492                                                                str(e), http_code=vimconn.HTTP_Conflict)
1493
1494                         try:
1495                             # for race condition ensure not already assigned
1496                             fip = self.neutron.show_floatingip(free_floating_ip)
1497                             if fip['floatingip']['port_id']:
1498                                 continue
1499                             # the vim_id key contains the neutron.port_id
1500                             self.neutron.update_floatingip(free_floating_ip,
1501                                                            {"floatingip": {"port_id": floating_network["vim_id"]}})
1502                             # for race condition ensure not re-assigned to other VM after 5 seconds
1503                             time.sleep(5)
1504                             fip = self.neutron.show_floatingip(free_floating_ip)
1505                             if fip['floatingip']['port_id'] != floating_network["vim_id"]:
1506                                 self.logger.error("floating_ip {} re-assigned to other port".format(free_floating_ip))
1507                                 continue
1508                             self.logger.debug("Assigned floating_ip {} to VM {}".format(free_floating_ip, server.id))
1509                             assigned = True
1510                         except Exception as e:
1511                             # openstack need some time after VM creation to assign an IP. So retry if fails
1512                             vm_status = self.nova.servers.get(server.id).status
1513                             if vm_status not in ('ACTIVE', 'ERROR'):
1514                                 if time.time() - vm_start_time < server_timeout:
1515                                     time.sleep(5)
1516                                     continue
1517                             elif floating_ip_retries > 0:
1518                                 floating_ip_retries -= 1
1519                                 continue
1520                             raise vimconn.VimConnException(
1521                                 "Cannot create floating_ip: {} {}".format(type(e).__name__, e),
1522                                 http_code=vimconn.HTTP_Conflict)
1523
1524                 except Exception as e:
1525                     if not floating_network['exit_on_floating_ip_error']:
1526                         self.logger.error("Cannot create floating_ip. %s", str(e))
1527                         continue
1528                     raise
1529
1530             return server.id, created_items
1531         # except nvExceptions.NotFound as e:
1532         #     error_value=-vimconn.HTTP_Not_Found
1533         #     error_text= "vm instance %s not found" % vm_id
1534         # except TypeError as e:
1535         #     raise vimconn.VimConnException(type(e).__name__ + ": "+  str(e), http_code=vimconn.HTTP_Bad_Request)
1536
1537         except Exception as e:
1538             server_id = None
1539             if server:
1540                 server_id = server.id
1541             try:
1542                 self.delete_vminstance(server_id, created_items)
1543             except Exception as e2:
1544                 self.logger.error("new_vminstance rollback fail {}".format(e2))
1545
1546             self._format_exception(e)
1547
1548     def get_vminstance(self, vm_id):
1549         """Returns the VM instance information from VIM"""
1550         # self.logger.debug("Getting VM from VIM")
1551         try:
1552             self._reload_connection()
1553             server = self.nova.servers.find(id=vm_id)
1554             # TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
1555             return server.to_dict()
1556         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound,
1557                 ConnectionError) as e:
1558             self._format_exception(e)
1559
1560     def get_vminstance_console(self, vm_id, console_type="vnc"):
1561         """
1562         Get a console for the virtual machine
1563         Params:
1564             vm_id: uuid of the VM
1565             console_type, can be:
1566                 "novnc" (by default), "xvpvnc" for VNC types,
1567                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
1568         Returns dict with the console parameters:
1569                 protocol: ssh, ftp, http, https, ...
1570                 server:   usually ip address
1571                 port:     the http, ssh, ... port
1572                 suffix:   extra text, e.g. the http path and query string
1573         """
1574         self.logger.debug("Getting VM CONSOLE from VIM")
1575         try:
1576             self._reload_connection()
1577             server = self.nova.servers.find(id=vm_id)
1578             if console_type is None or console_type == "novnc":
1579                 console_dict = server.get_vnc_console("novnc")
1580             elif console_type == "xvpvnc":
1581                 console_dict = server.get_vnc_console(console_type)
1582             elif console_type == "rdp-html5":
1583                 console_dict = server.get_rdp_console(console_type)
1584             elif console_type == "spice-html5":
1585                 console_dict = server.get_spice_console(console_type)
1586             else:
1587                 raise vimconn.VimConnException("console type '{}' not allowed".format(console_type),
1588                                                http_code=vimconn.HTTP_Bad_Request)
1589
1590             console_dict1 = console_dict.get("console")
1591             if console_dict1:
1592                 console_url = console_dict1.get("url")
1593                 if console_url:
1594                     # parse console_url
1595                     protocol_index = console_url.find("//")
1596                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1597                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1598                     if protocol_index < 0 or port_index < 0 or suffix_index < 0:
1599                         return -vimconn.HTTP_Internal_Server_Error, "Unexpected response from VIM"
1600                     console_dict = {"protocol": console_url[0:protocol_index],
1601                                     "server": console_url[protocol_index+2:port_index],
1602                                     "port": console_url[port_index:suffix_index],
1603                                     "suffix": console_url[suffix_index+1:]
1604                                     }
1605                     protocol_index += 2
1606                     return console_dict
1607             raise vimconn.VimConnUnexpectedResponse("Unexpected response from VIM")
1608
1609         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException,
1610                 nvExceptions.BadRequest, ConnectionError) as e:
1611             self._format_exception(e)
1612
1613     def delete_vminstance(self, vm_id, created_items=None):
1614         """Removes a VM instance from VIM. Returns the old identifier
1615         """
1616         # print "osconnector: Getting VM from VIM"
1617         if created_items is None:
1618             created_items = {}
1619         try:
1620             self._reload_connection()
1621             # delete VM ports attached to this networks before the virtual machine
1622             for k, v in created_items.items():
1623                 if not v:  # skip already deleted
1624                     continue
1625                 try:
1626                     k_item, _, k_id = k.partition(":")
1627                     if k_item == "port":
1628                         self.neutron.delete_port(k_id)
1629                 except Exception as e:
1630                     self.logger.error("Error deleting port: {}: {}".format(type(e).__name__, e))
1631
1632             # #commented because detaching the volumes makes the servers.delete not work properly ?!?
1633             # #dettach volumes attached
1634             # server = self.nova.servers.get(vm_id)
1635             # volumes_attached_dict = server._info['os-extended-volumes:volumes_attached']   #volume['id']
1636             # #for volume in volumes_attached_dict:
1637             # #    self.cinder.volumes.detach(volume['id'])
1638
1639             if vm_id:
1640                 self.nova.servers.delete(vm_id)
1641
1642             # delete volumes. Although having detached, they should have in active status before deleting
1643             # we ensure in this loop
1644             keep_waiting = True
1645             elapsed_time = 0
1646             while keep_waiting and elapsed_time < volume_timeout:
1647                 keep_waiting = False
1648                 for k, v in created_items.items():
1649                     if not v:  # skip already deleted
1650                         continue
1651                     try:
1652                         k_item, _, k_id = k.partition(":")
1653                         if k_item == "volume":
1654                             if self.cinder.volumes.get(k_id).status != 'available':
1655                                 keep_waiting = True
1656                             else:
1657                                 self.cinder.volumes.delete(k_id)
1658                                 created_items[k] = None
1659                         elif k_item == "floating_ip":  # floating ip
1660                             self.neutron.delete_floatingip(k_id)
1661                             created_items[k] = None
1662
1663                     except Exception as e:
1664                         self.logger.error("Error deleting {}: {}".format(k, e))
1665                 if keep_waiting:
1666                     time.sleep(1)
1667                     elapsed_time += 1
1668             return None
1669         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException,
1670                 ConnectionError) as e:
1671             self._format_exception(e)
1672
1673     def refresh_vms_status(self, vm_list):
1674         """Get the status of the virtual machines and their interfaces/ports
1675            Params: the list of VM identifiers
1676            Returns a dictionary with:
1677                 vm_id:          #VIM id of this Virtual Machine
1678                     status:     #Mandatory. Text with one of:
1679                                 #  DELETED (not found at vim)
1680                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1681                                 #  OTHER (Vim reported other status not understood)
1682                                 #  ERROR (VIM indicates an ERROR status)
1683                                 #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
1684                                 #  CREATING (on building process), ERROR
1685                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
1686                                 #
1687                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
1688                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
1689                     interfaces:
1690                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
1691                         mac_address:      #Text format XX:XX:XX:XX:XX:XX
1692                         vim_net_id:       #network id where this interface is connected
1693                         vim_interface_id: #interface/port VIM id
1694                         ip_address:       #null, or text with IPv4, IPv6 address
1695                         compute_node:     #identification of compute node where PF,VF interface is allocated
1696                         pci:              #PCI address of the NIC that hosts the PF,VF
1697                         vlan:             #physical VLAN used for VF
1698         """
1699         vm_dict = {}
1700         self.logger.debug("refresh_vms status: Getting tenant VM instance information from VIM")
1701         for vm_id in vm_list:
1702             vm = {}
1703             try:
1704                 vm_vim = self.get_vminstance(vm_id)
1705                 if vm_vim['status'] in vmStatus2manoFormat:
1706                     vm['status'] = vmStatus2manoFormat[vm_vim['status']]
1707                 else:
1708                     vm['status'] = "OTHER"
1709                     vm['error_msg'] = "VIM status reported " + vm_vim['status']
1710                 vm_vim.pop("OS-EXT-SRV-ATTR:user_data", None)
1711                 vm_vim.pop("user_data", None)
1712                 vm['vim_info'] = self.serialize(vm_vim)
1713
1714                 vm["interfaces"] = []
1715                 if vm_vim.get('fault'):
1716                     vm['error_msg'] = str(vm_vim['fault'])
1717                 # get interfaces
1718                 try:
1719                     self._reload_connection()
1720                     port_dict = self.neutron.list_ports(device_id=vm_id)
1721                     for port in port_dict["ports"]:
1722                         interface = {}
1723                         interface['vim_info'] = self.serialize(port)
1724                         interface["mac_address"] = port.get("mac_address")
1725                         interface["vim_net_id"] = port["network_id"]
1726                         interface["vim_interface_id"] = port["id"]
1727                         # check if OS-EXT-SRV-ATTR:host is there,
1728                         # in case of non-admin credentials, it will be missing
1729                         if vm_vim.get('OS-EXT-SRV-ATTR:host'):
1730                             interface["compute_node"] = vm_vim['OS-EXT-SRV-ATTR:host']
1731                         interface["pci"] = None
1732
1733                         # check if binding:profile is there,
1734                         # in case of non-admin credentials, it will be missing
1735                         if port.get('binding:profile'):
1736                             if port['binding:profile'].get('pci_slot'):
1737                                 # TODO: At the moment sr-iov pci addresses are converted to PF pci addresses by setting
1738                                 #  the slot to 0x00
1739                                 # TODO: This is just a workaround valid for niantinc. Find a better way to do so
1740                                 #   CHANGE DDDD:BB:SS.F to DDDD:BB:00.(F%2)   assuming there are 2 ports per nic
1741                                 pci = port['binding:profile']['pci_slot']
1742                                 # interface["pci"] = pci[:-4] + "00." + str(int(pci[-1]) % 2)
1743                                 interface["pci"] = pci
1744                         interface["vlan"] = None
1745                         if port.get('binding:vif_details'):
1746                             interface["vlan"] = port['binding:vif_details'].get('vlan')
1747                         # Get vlan from network in case not present in port for those old openstacks and cases where
1748                         # it is needed vlan at PT
1749                         if not interface["vlan"]:
1750                             # if network is of type vlan and port is of type direct (sr-iov) then set vlan id
1751                             network = self.neutron.show_network(port["network_id"])
1752                             if network['network'].get('provider:network_type') == 'vlan':
1753                                 # and port.get("binding:vnic_type") in ("direct", "direct-physical"):
1754                                 interface["vlan"] = network['network'].get('provider:segmentation_id')
1755                         ips = []
1756                         # look for floating ip address
1757                         try:
1758                             floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
1759                             if floating_ip_dict.get("floatingips"):
1760                                 ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address"))
1761                         except Exception:
1762                             pass
1763
1764                         for subnet in port["fixed_ips"]:
1765                             ips.append(subnet["ip_address"])
1766                         interface["ip_address"] = ";".join(ips)
1767                         vm["interfaces"].append(interface)
1768                 except Exception as e:
1769                     self.logger.error("Error getting vm interface information {}: {}".format(type(e).__name__, e),
1770                                       exc_info=True)
1771             except vimconn.VimConnNotFoundException as e:
1772                 self.logger.error("Exception getting vm status: %s", str(e))
1773                 vm['status'] = "DELETED"
1774                 vm['error_msg'] = str(e)
1775             except vimconn.VimConnException as e:
1776                 self.logger.error("Exception getting vm status: %s", str(e))
1777                 vm['status'] = "VIM_ERROR"
1778                 vm['error_msg'] = str(e)
1779             vm_dict[vm_id] = vm
1780         return vm_dict
1781
1782     def action_vminstance(self, vm_id, action_dict, created_items={}):
1783         """Send and action over a VM instance from VIM
1784         Returns None or the console dict if the action was successfully sent to the VIM"""
1785         self.logger.debug("Action over VM '%s': %s", vm_id, str(action_dict))
1786         try:
1787             self._reload_connection()
1788             server = self.nova.servers.find(id=vm_id)
1789             if "start" in action_dict:
1790                 if action_dict["start"] == "rebuild":
1791                     server.rebuild()
1792                 else:
1793                     if server.status == "PAUSED":
1794                         server.unpause()
1795                     elif server.status == "SUSPENDED":
1796                         server.resume()
1797                     elif server.status == "SHUTOFF":
1798                         server.start()
1799             elif "pause" in action_dict:
1800                 server.pause()
1801             elif "resume" in action_dict:
1802                 server.resume()
1803             elif "shutoff" in action_dict or "shutdown" in action_dict:
1804                 server.stop()
1805             elif "forceOff" in action_dict:
1806                 server.stop()  # TODO
1807             elif "terminate" in action_dict:
1808                 server.delete()
1809             elif "createImage" in action_dict:
1810                 server.create_image()
1811                 # "path":path_schema,
1812                 # "description":description_schema,
1813                 # "name":name_schema,
1814                 # "metadata":metadata_schema,
1815                 # "imageRef": id_schema,
1816                 # "disk": {"oneOf":[{"type": "null"}, {"type":"string"}] },
1817             elif "rebuild" in action_dict:
1818                 server.rebuild(server.image['id'])
1819             elif "reboot" in action_dict:
1820                 server.reboot()   # reboot_type='SOFT'
1821             elif "console" in action_dict:
1822                 console_type = action_dict["console"]
1823                 if console_type is None or console_type == "novnc":
1824                     console_dict = server.get_vnc_console("novnc")
1825                 elif console_type == "xvpvnc":
1826                     console_dict = server.get_vnc_console(console_type)
1827                 elif console_type == "rdp-html5":
1828                     console_dict = server.get_rdp_console(console_type)
1829                 elif console_type == "spice-html5":
1830                     console_dict = server.get_spice_console(console_type)
1831                 else:
1832                     raise vimconn.VimConnException("console type '{}' not allowed".format(console_type),
1833                                                    http_code=vimconn.HTTP_Bad_Request)
1834                 try:
1835                     console_url = console_dict["console"]["url"]
1836                     # parse console_url
1837                     protocol_index = console_url.find("//")
1838                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1839                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1840                     if protocol_index < 0 or port_index < 0 or suffix_index < 0:
1841                         raise vimconn.VimConnException("Unexpected response from VIM " + str(console_dict))
1842                     console_dict2 = {"protocol": console_url[0:protocol_index],
1843                                      "server": console_url[protocol_index+2: port_index],
1844                                      "port": int(console_url[port_index+1: suffix_index]),
1845                                      "suffix": console_url[suffix_index+1:]
1846                                      }
1847                     return console_dict2
1848                 except Exception:
1849                     raise vimconn.VimConnException("Unexpected response from VIM " + str(console_dict))
1850
1851             return None
1852         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound,
1853                 ConnectionError) as e:
1854             self._format_exception(e)
1855         # TODO insert exception vimconn.HTTP_Unauthorized
1856
1857     # ###### VIO Specific Changes #########
1858     def _generate_vlanID(self):
1859         """
1860          Method to get unused vlanID
1861             Args:
1862                 None
1863             Returns:
1864                 vlanID
1865         """
1866         # Get used VLAN IDs
1867         usedVlanIDs = []
1868         networks = self.get_network_list()
1869         for net in networks:
1870             if net.get('provider:segmentation_id'):
1871                 usedVlanIDs.append(net.get('provider:segmentation_id'))
1872         used_vlanIDs = set(usedVlanIDs)
1873
1874         # find unused VLAN ID
1875         for vlanID_range in self.config.get('dataplane_net_vlan_range'):
1876             try:
1877                 start_vlanid, end_vlanid = map(int, vlanID_range.replace(" ", "").split("-"))
1878                 for vlanID in range(start_vlanid, end_vlanid + 1):
1879                     if vlanID not in used_vlanIDs:
1880                         return vlanID
1881             except Exception as exp:
1882                 raise vimconn.VimConnException("Exception {} occurred while generating VLAN ID.".format(exp))
1883         else:
1884             raise vimconn.VimConnConflictException(
1885                 "Unable to create the SRIOV VLAN network. All given Vlan IDs {} are in use.".format(
1886                     self.config.get('dataplane_net_vlan_range')))
1887
1888     def _generate_multisegment_vlanID(self):
1889         """
1890          Method to get unused vlanID
1891             Args:
1892                 None
1893             Returns:
1894                 vlanID
1895         """
1896         # Get used VLAN IDs
1897         usedVlanIDs = []
1898         networks = self.get_network_list()
1899         for net in networks:
1900             if net.get('provider:network_type') == "vlan" and net.get('provider:segmentation_id'):
1901                 usedVlanIDs.append(net.get('provider:segmentation_id'))
1902             elif net.get('segments'):
1903                 for segment in net.get('segments'):
1904                     if segment.get('provider:network_type') == "vlan" and segment.get('provider:segmentation_id'):
1905                         usedVlanIDs.append(segment.get('provider:segmentation_id'))
1906         used_vlanIDs = set(usedVlanIDs)
1907
1908         # find unused VLAN ID
1909         for vlanID_range in self.config.get('multisegment_vlan_range'):
1910             try:
1911                 start_vlanid, end_vlanid = map(int, vlanID_range.replace(" ", "").split("-"))
1912                 for vlanID in range(start_vlanid, end_vlanid + 1):
1913                     if vlanID not in used_vlanIDs:
1914                         return vlanID
1915             except Exception as exp:
1916                 raise vimconn.VimConnException("Exception {} occurred while generating VLAN ID.".format(exp))
1917         else:
1918             raise vimconn.VimConnConflictException(
1919                 "Unable to create the VLAN segment. All VLAN IDs {} are in use.".format(
1920                     self.config.get('multisegment_vlan_range')))
1921
1922     def _validate_vlan_ranges(self, input_vlan_range, text_vlan_range):
1923         """
1924         Method to validate user given vlanID ranges
1925             Args:  None
1926             Returns: None
1927         """
1928         for vlanID_range in input_vlan_range:
1929             vlan_range = vlanID_range.replace(" ", "")
1930             # validate format
1931             vlanID_pattern = r'(\d)*-(\d)*$'
1932             match_obj = re.match(vlanID_pattern, vlan_range)
1933             if not match_obj:
1934                 raise vimconn.VimConnConflictException(
1935                     "Invalid VLAN range for {}: {}.You must provide '{}' in format [start_ID - end_ID].".format(
1936                         text_vlan_range, vlanID_range, text_vlan_range))
1937
1938             start_vlanid, end_vlanid = map(int, vlan_range.split("-"))
1939             if start_vlanid <= 0:
1940                 raise vimconn.VimConnConflictException(
1941                     "Invalid VLAN range for {}: {}. Start ID can not be zero. For VLAN "
1942                     "networks valid IDs are 1 to 4094 ".format(text_vlan_range, vlanID_range))
1943             if end_vlanid > 4094:
1944                 raise vimconn.VimConnConflictException(
1945                     "Invalid VLAN range for {}: {}. End VLAN ID can not be greater than 4094. For VLAN "
1946                     "networks valid IDs are 1 to 4094 ".format(text_vlan_range, vlanID_range))
1947
1948             if start_vlanid > end_vlanid:
1949                 raise vimconn.VimConnConflictException(
1950                     "Invalid VLAN range for {}: {}. You must provide '{}' in format start_ID - end_ID and "
1951                     "start_ID < end_ID ".format(text_vlan_range, vlanID_range, text_vlan_range))
1952
1953     # NOT USED FUNCTIONS
1954
1955     def new_external_port(self, port_data):
1956         """Adds a external port to VIM
1957            Returns the port identifier"""
1958         # TODO openstack if needed
1959         return -vimconn.HTTP_Internal_Server_Error, "osconnector.new_external_port() not implemented"
1960
1961     def connect_port_network(self, port_id, network_id, admin=False):
1962         """Connects a external port to a network
1963             Returns status code of the VIM response"""
1964         # TODO openstack if needed
1965         return -vimconn.HTTP_Internal_Server_Error, "osconnector.connect_port_network() not implemented"
1966
1967     def new_user(self, user_name, user_passwd, tenant_id=None):
1968         """Adds a new user to openstack VIM
1969            Returns the user identifier"""
1970         self.logger.debug("osconnector: Adding a new user to VIM")
1971         try:
1972             self._reload_connection()
1973             user = self.keystone.users.create(user_name, password=user_passwd, default_project=tenant_id)
1974             # self.keystone.tenants.add_user(self.k_creds["username"], #role)
1975             return user.id
1976         except ksExceptions.ConnectionError as e:
1977             error_value = -vimconn.HTTP_Bad_Request
1978             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
1979         except ksExceptions.ClientException as e:  # TODO remove
1980             error_value = -vimconn.HTTP_Bad_Request
1981             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
1982         # TODO insert exception vimconn.HTTP_Unauthorized
1983         # if reaching here is because an exception
1984         self.logger.debug("new_user " + error_text)
1985         return error_value, error_text
1986
1987     def delete_user(self, user_id):
1988         """Delete a user from openstack VIM
1989            Returns the user identifier"""
1990         if self.debug:
1991             print("osconnector: Deleting  a  user from VIM")
1992         try:
1993             self._reload_connection()
1994             self.keystone.users.delete(user_id)
1995             return 1, user_id
1996         except ksExceptions.ConnectionError as e:
1997             error_value = -vimconn.HTTP_Bad_Request
1998             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
1999         except ksExceptions.NotFound as e:
2000             error_value = -vimconn.HTTP_Not_Found
2001             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
2002         except ksExceptions.ClientException as e:  # TODO remove
2003             error_value = -vimconn.HTTP_Bad_Request
2004             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
2005         # TODO insert exception vimconn.HTTP_Unauthorized
2006         # if reaching here is because an exception
2007         self.logger.debug("delete_tenant " + error_text)
2008         return error_value, error_text
2009
2010     def get_hosts_info(self):
2011         """Get the information of deployed hosts
2012         Returns the hosts content"""
2013         if self.debug:
2014             print("osconnector: Getting Host info from VIM")
2015         try:
2016             h_list = []
2017             self._reload_connection()
2018             hypervisors = self.nova.hypervisors.list()
2019             for hype in hypervisors:
2020                 h_list.append(hype.to_dict())
2021             return 1, {"hosts": h_list}
2022         except nvExceptions.NotFound as e:
2023             error_value = -vimconn.HTTP_Not_Found
2024             error_text = (str(e) if len(e.args) == 0 else str(e.args[0]))
2025         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
2026             error_value = -vimconn.HTTP_Bad_Request
2027             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
2028         # TODO insert exception vimconn.HTTP_Unauthorized
2029         # if reaching here is because an exception
2030         self.logger.debug("get_hosts_info " + error_text)
2031         return error_value, error_text
2032
2033     def get_hosts(self, vim_tenant):
2034         """Get the hosts and deployed instances
2035         Returns the hosts content"""
2036         r, hype_dict = self.get_hosts_info()
2037         if r < 0:
2038             return r, hype_dict
2039         hypervisors = hype_dict["hosts"]
2040         try:
2041             servers = self.nova.servers.list()
2042             for hype in hypervisors:
2043                 for server in servers:
2044                     if server.to_dict()['OS-EXT-SRV-ATTR:hypervisor_hostname'] == hype['hypervisor_hostname']:
2045                         if 'vm' in hype:
2046                             hype['vm'].append(server.id)
2047                         else:
2048                             hype['vm'] = [server.id]
2049             return 1, hype_dict
2050         except nvExceptions.NotFound as e:
2051             error_value = -vimconn.HTTP_Not_Found
2052             error_text = (str(e) if len(e.args) == 0 else str(e.args[0]))
2053         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
2054             error_value = -vimconn.HTTP_Bad_Request
2055             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
2056         # TODO insert exception vimconn.HTTP_Unauthorized
2057         # if reaching here is because an exception
2058         self.logger.debug("get_hosts " + error_text)
2059         return error_value, error_text
2060
2061     def new_classification(self, name, ctype, definition):
2062         self.logger.debug('Adding a new (Traffic) Classification to VIM, named %s', name)
2063         try:
2064             new_class = None
2065             self._reload_connection()
2066             if ctype not in supportedClassificationTypes:
2067                 raise vimconn.VimConnNotSupportedException(
2068                     'OpenStack VIM connector does not support provided Classification Type {}, supported ones are: '
2069                     '{}'.format(ctype, supportedClassificationTypes))
2070             if not self._validate_classification(ctype, definition):
2071                 raise vimconn.VimConnException(
2072                     'Incorrect Classification definition '
2073                     'for the type specified.')
2074             classification_dict = definition
2075             classification_dict['name'] = name
2076
2077             new_class = self.neutron.create_sfc_flow_classifier(
2078                 {'flow_classifier': classification_dict})
2079             return new_class['flow_classifier']['id']
2080         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2081                 neExceptions.NeutronException, ConnectionError) as e:
2082             self.logger.error(
2083                 'Creation of Classification failed.')
2084             self._format_exception(e)
2085
2086     def get_classification(self, class_id):
2087         self.logger.debug(" Getting Classification %s from VIM", class_id)
2088         filter_dict = {"id": class_id}
2089         class_list = self.get_classification_list(filter_dict)
2090         if len(class_list) == 0:
2091             raise vimconn.VimConnNotFoundException(
2092                 "Classification '{}' not found".format(class_id))
2093         elif len(class_list) > 1:
2094             raise vimconn.VimConnConflictException(
2095                 "Found more than one Classification with this criteria")
2096         classification = class_list[0]
2097         return classification
2098
2099     def get_classification_list(self, filter_dict={}):
2100         self.logger.debug("Getting Classifications from VIM filter: '%s'",
2101                           str(filter_dict))
2102         try:
2103             filter_dict_os = filter_dict.copy()
2104             self._reload_connection()
2105             if self.api_version3 and "tenant_id" in filter_dict_os:
2106                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
2107             classification_dict = self.neutron.list_sfc_flow_classifiers(
2108                 **filter_dict_os)
2109             classification_list = classification_dict["flow_classifiers"]
2110             self.__classification_os2mano(classification_list)
2111             return classification_list
2112         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2113                 neExceptions.NeutronException, ConnectionError) as e:
2114             self._format_exception(e)
2115
2116     def delete_classification(self, class_id):
2117         self.logger.debug("Deleting Classification '%s' from VIM", class_id)
2118         try:
2119             self._reload_connection()
2120             self.neutron.delete_sfc_flow_classifier(class_id)
2121             return class_id
2122         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
2123                 ksExceptions.ClientException, neExceptions.NeutronException,
2124                 ConnectionError) as e:
2125             self._format_exception(e)
2126
2127     def new_sfi(self, name, ingress_ports, egress_ports, sfc_encap=True):
2128         self.logger.debug("Adding a new Service Function Instance to VIM, named '%s'", name)
2129         try:
2130             new_sfi = None
2131             self._reload_connection()
2132             correlation = None
2133             if sfc_encap:
2134                 correlation = 'nsh'
2135             if len(ingress_ports) != 1:
2136                 raise vimconn.VimConnNotSupportedException(
2137                     "OpenStack VIM connector can only have "
2138                     "1 ingress port per SFI")
2139             if len(egress_ports) != 1:
2140                 raise vimconn.VimConnNotSupportedException(
2141                     "OpenStack VIM connector can only have "
2142                     "1 egress port per SFI")
2143             sfi_dict = {'name': name,
2144                         'ingress': ingress_ports[0],
2145                         'egress': egress_ports[0],
2146                         'service_function_parameters': {
2147                             'correlation': correlation}}
2148             new_sfi = self.neutron.create_sfc_port_pair({'port_pair': sfi_dict})
2149             return new_sfi['port_pair']['id']
2150         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2151                 neExceptions.NeutronException, ConnectionError) as e:
2152             if new_sfi:
2153                 try:
2154                     self.neutron.delete_sfc_port_pair(
2155                         new_sfi['port_pair']['id'])
2156                 except Exception:
2157                     self.logger.error(
2158                         'Creation of Service Function Instance failed, with '
2159                         'subsequent deletion failure as well.')
2160             self._format_exception(e)
2161
2162     def get_sfi(self, sfi_id):
2163         self.logger.debug('Getting Service Function Instance %s from VIM', sfi_id)
2164         filter_dict = {"id": sfi_id}
2165         sfi_list = self.get_sfi_list(filter_dict)
2166         if len(sfi_list) == 0:
2167             raise vimconn.VimConnNotFoundException("Service Function Instance '{}' not found".format(sfi_id))
2168         elif len(sfi_list) > 1:
2169             raise vimconn.VimConnConflictException(
2170                 'Found more than one Service Function Instance '
2171                 'with this criteria')
2172         sfi = sfi_list[0]
2173         return sfi
2174
2175     def get_sfi_list(self, filter_dict={}):
2176         self.logger.debug("Getting Service Function Instances from VIM filter: '%s'", str(filter_dict))
2177         try:
2178             self._reload_connection()
2179             filter_dict_os = filter_dict.copy()
2180             if self.api_version3 and "tenant_id" in filter_dict_os:
2181                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
2182             sfi_dict = self.neutron.list_sfc_port_pairs(**filter_dict_os)
2183             sfi_list = sfi_dict["port_pairs"]
2184             self.__sfi_os2mano(sfi_list)
2185             return sfi_list
2186         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2187                 neExceptions.NeutronException, ConnectionError) as e:
2188             self._format_exception(e)
2189
2190     def delete_sfi(self, sfi_id):
2191         self.logger.debug("Deleting Service Function Instance '%s' "
2192                           "from VIM", sfi_id)
2193         try:
2194             self._reload_connection()
2195             self.neutron.delete_sfc_port_pair(sfi_id)
2196             return sfi_id
2197         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
2198                 ksExceptions.ClientException, neExceptions.NeutronException,
2199                 ConnectionError) as e:
2200             self._format_exception(e)
2201
2202     def new_sf(self, name, sfis, sfc_encap=True):
2203         self.logger.debug("Adding a new Service Function to VIM, named '%s'", name)
2204         try:
2205             new_sf = None
2206             self._reload_connection()
2207             # correlation = None
2208             # if sfc_encap:
2209             #     correlation = 'nsh'
2210             for instance in sfis:
2211                 sfi = self.get_sfi(instance)
2212                 if sfi.get('sfc_encap') != sfc_encap:
2213                     raise vimconn.VimConnNotSupportedException(
2214                         "OpenStack VIM connector requires all SFIs of the "
2215                         "same SF to share the same SFC Encapsulation")
2216             sf_dict = {'name': name,
2217                        'port_pairs': sfis}
2218             new_sf = self.neutron.create_sfc_port_pair_group({
2219                 'port_pair_group': sf_dict})
2220             return new_sf['port_pair_group']['id']
2221         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2222                 neExceptions.NeutronException, ConnectionError) as e:
2223             if new_sf:
2224                 try:
2225                     self.neutron.delete_sfc_port_pair_group(
2226                         new_sf['port_pair_group']['id'])
2227                 except Exception:
2228                     self.logger.error(
2229                         'Creation of Service Function failed, with '
2230                         'subsequent deletion failure as well.')
2231             self._format_exception(e)
2232
2233     def get_sf(self, sf_id):
2234         self.logger.debug("Getting Service Function %s from VIM", sf_id)
2235         filter_dict = {"id": sf_id}
2236         sf_list = self.get_sf_list(filter_dict)
2237         if len(sf_list) == 0:
2238             raise vimconn.VimConnNotFoundException(
2239                 "Service Function '{}' not found".format(sf_id))
2240         elif len(sf_list) > 1:
2241             raise vimconn.VimConnConflictException(
2242                 "Found more than one Service Function with this criteria")
2243         sf = sf_list[0]
2244         return sf
2245
2246     def get_sf_list(self, filter_dict={}):
2247         self.logger.debug("Getting Service Function from VIM filter: '%s'",
2248                           str(filter_dict))
2249         try:
2250             self._reload_connection()
2251             filter_dict_os = filter_dict.copy()
2252             if self.api_version3 and "tenant_id" in filter_dict_os:
2253                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
2254             sf_dict = self.neutron.list_sfc_port_pair_groups(**filter_dict_os)
2255             sf_list = sf_dict["port_pair_groups"]
2256             self.__sf_os2mano(sf_list)
2257             return sf_list
2258         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2259                 neExceptions.NeutronException, ConnectionError) as e:
2260             self._format_exception(e)
2261
2262     def delete_sf(self, sf_id):
2263         self.logger.debug("Deleting Service Function '%s' from VIM", sf_id)
2264         try:
2265             self._reload_connection()
2266             self.neutron.delete_sfc_port_pair_group(sf_id)
2267             return sf_id
2268         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
2269                 ksExceptions.ClientException, neExceptions.NeutronException,
2270                 ConnectionError) as e:
2271             self._format_exception(e)
2272
2273     def new_sfp(self, name, classifications, sfs, sfc_encap=True, spi=None):
2274         self.logger.debug("Adding a new Service Function Path to VIM, named '%s'", name)
2275         try:
2276             new_sfp = None
2277             self._reload_connection()
2278             # In networking-sfc the MPLS encapsulation is legacy
2279             # should be used when no full SFC Encapsulation is intended
2280             correlation = 'mpls'
2281             if sfc_encap:
2282                 correlation = 'nsh'
2283             sfp_dict = {'name': name,
2284                         'flow_classifiers': classifications,
2285                         'port_pair_groups': sfs,
2286                         'chain_parameters': {'correlation': correlation}}
2287             if spi:
2288                 sfp_dict['chain_id'] = spi
2289             new_sfp = self.neutron.create_sfc_port_chain({'port_chain': sfp_dict})
2290             return new_sfp["port_chain"]["id"]
2291         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2292                 neExceptions.NeutronException, ConnectionError) as e:
2293             if new_sfp:
2294                 try:
2295                     self.neutron.delete_sfc_port_chain(new_sfp['port_chain']['id'])
2296                 except Exception:
2297                     self.logger.error(
2298                         'Creation of Service Function Path failed, with '
2299                         'subsequent deletion failure as well.')
2300             self._format_exception(e)
2301
2302     def get_sfp(self, sfp_id):
2303         self.logger.debug(" Getting Service Function Path %s from VIM", sfp_id)
2304         filter_dict = {"id": sfp_id}
2305         sfp_list = self.get_sfp_list(filter_dict)
2306         if len(sfp_list) == 0:
2307             raise vimconn.VimConnNotFoundException(
2308                 "Service Function Path '{}' not found".format(sfp_id))
2309         elif len(sfp_list) > 1:
2310             raise vimconn.VimConnConflictException(
2311                 "Found more than one Service Function Path with this criteria")
2312         sfp = sfp_list[0]
2313         return sfp
2314
2315     def get_sfp_list(self, filter_dict={}):
2316         self.logger.debug("Getting Service Function Paths from VIM filter: '%s'", str(filter_dict))
2317         try:
2318             self._reload_connection()
2319             filter_dict_os = filter_dict.copy()
2320             if self.api_version3 and "tenant_id" in filter_dict_os:
2321                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
2322             sfp_dict = self.neutron.list_sfc_port_chains(**filter_dict_os)
2323             sfp_list = sfp_dict["port_chains"]
2324             self.__sfp_os2mano(sfp_list)
2325             return sfp_list
2326         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2327                 neExceptions.NeutronException, ConnectionError) as e:
2328             self._format_exception(e)
2329
2330     def delete_sfp(self, sfp_id):
2331         self.logger.debug("Deleting Service Function Path '%s' from VIM", sfp_id)
2332         try:
2333             self._reload_connection()
2334             self.neutron.delete_sfc_port_chain(sfp_id)
2335             return sfp_id
2336         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
2337                 ksExceptions.ClientException, neExceptions.NeutronException,
2338                 ConnectionError) as e:
2339             self._format_exception(e)
2340
2341     def refresh_sfps_status(self, sfp_list):
2342         """Get the status of the service function path
2343            Params: the list of sfp identifiers
2344            Returns a dictionary with:
2345                 vm_id:          #VIM id of this service function path
2346                     status:     #Mandatory. Text with one of:
2347                                 #  DELETED (not found at vim)
2348                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
2349                                 #  OTHER (Vim reported other status not understood)
2350                                 #  ERROR (VIM indicates an ERROR status)
2351                                 #  ACTIVE,
2352                                 #  CREATING (on building process)
2353                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
2354                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)F
2355         """
2356         sfp_dict = {}
2357         self.logger.debug("refresh_sfps status: Getting tenant SFP information from VIM")
2358         for sfp_id in sfp_list:
2359             sfp = {}
2360             try:
2361                 sfp_vim = self.get_sfp(sfp_id)
2362                 if sfp_vim['spi']:
2363                     sfp['status'] = vmStatus2manoFormat['ACTIVE']
2364                 else:
2365                     sfp['status'] = "OTHER"
2366                     sfp['error_msg'] = "VIM status reported " + sfp['status']
2367
2368                 sfp['vim_info'] = self.serialize(sfp_vim)
2369
2370                 if sfp_vim.get('fault'):
2371                     sfp['error_msg'] = str(sfp_vim['fault'])
2372
2373             except vimconn.VimConnNotFoundException as e:
2374                 self.logger.error("Exception getting sfp status: %s", str(e))
2375                 sfp['status'] = "DELETED"
2376                 sfp['error_msg'] = str(e)
2377             except vimconn.VimConnException as e:
2378                 self.logger.error("Exception getting sfp status: %s", str(e))
2379                 sfp['status'] = "VIM_ERROR"
2380                 sfp['error_msg'] = str(e)
2381             sfp_dict[sfp_id] = sfp
2382         return sfp_dict
2383
2384     def refresh_sfis_status(self, sfi_list):
2385         """Get the status of the service function instances
2386            Params: the list of sfi identifiers
2387            Returns a dictionary with:
2388                 vm_id:          #VIM id of this service function instance
2389                     status:     #Mandatory. Text with one of:
2390                                 #  DELETED (not found at vim)
2391                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
2392                                 #  OTHER (Vim reported other status not understood)
2393                                 #  ERROR (VIM indicates an ERROR status)
2394                                 #  ACTIVE,
2395                                 #  CREATING (on building process)
2396                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
2397                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
2398         """
2399         sfi_dict = {}
2400         self.logger.debug("refresh_sfis status: Getting tenant sfi information from VIM")
2401         for sfi_id in sfi_list:
2402             sfi = {}
2403             try:
2404                 sfi_vim = self.get_sfi(sfi_id)
2405                 if sfi_vim:
2406                     sfi['status'] = vmStatus2manoFormat['ACTIVE']
2407                 else:
2408                     sfi['status'] = "OTHER"
2409                     sfi['error_msg'] = "VIM status reported " + sfi['status']
2410
2411                 sfi['vim_info'] = self.serialize(sfi_vim)
2412
2413                 if sfi_vim.get('fault'):
2414                     sfi['error_msg'] = str(sfi_vim['fault'])
2415
2416             except vimconn.VimConnNotFoundException as e:
2417                 self.logger.error("Exception getting sfi status: %s", str(e))
2418                 sfi['status'] = "DELETED"
2419                 sfi['error_msg'] = str(e)
2420             except vimconn.VimConnException as e:
2421                 self.logger.error("Exception getting sfi status: %s", str(e))
2422                 sfi['status'] = "VIM_ERROR"
2423                 sfi['error_msg'] = str(e)
2424             sfi_dict[sfi_id] = sfi
2425         return sfi_dict
2426
2427     def refresh_sfs_status(self, sf_list):
2428         """Get the status of the service functions
2429            Params: the list of sf identifiers
2430            Returns a dictionary with:
2431                 vm_id:          #VIM id of this service function
2432                     status:     #Mandatory. Text with one of:
2433                                 #  DELETED (not found at vim)
2434                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
2435                                 #  OTHER (Vim reported other status not understood)
2436                                 #  ERROR (VIM indicates an ERROR status)
2437                                 #  ACTIVE,
2438                                 #  CREATING (on building process)
2439                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
2440                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
2441         """
2442         sf_dict = {}
2443         self.logger.debug("refresh_sfs status: Getting tenant sf information from VIM")
2444         for sf_id in sf_list:
2445             sf = {}
2446             try:
2447                 sf_vim = self.get_sf(sf_id)
2448                 if sf_vim:
2449                     sf['status'] = vmStatus2manoFormat['ACTIVE']
2450                 else:
2451                     sf['status'] = "OTHER"
2452                     sf['error_msg'] = "VIM status reported " + sf_vim['status']
2453
2454                 sf['vim_info'] = self.serialize(sf_vim)
2455
2456                 if sf_vim.get('fault'):
2457                     sf['error_msg'] = str(sf_vim['fault'])
2458
2459             except vimconn.VimConnNotFoundException as e:
2460                 self.logger.error("Exception getting sf status: %s", str(e))
2461                 sf['status'] = "DELETED"
2462                 sf['error_msg'] = str(e)
2463             except vimconn.VimConnException as e:
2464                 self.logger.error("Exception getting sf status: %s", str(e))
2465                 sf['status'] = "VIM_ERROR"
2466                 sf['error_msg'] = str(e)
2467             sf_dict[sf_id] = sf
2468         return sf_dict
2469
2470     def refresh_classifications_status(self, classification_list):
2471         """Get the status of the classifications
2472            Params: the list of classification identifiers
2473            Returns a dictionary with:
2474                 vm_id:          #VIM id of this classifier
2475                     status:     #Mandatory. Text with one of:
2476                                 #  DELETED (not found at vim)
2477                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
2478                                 #  OTHER (Vim reported other status not understood)
2479                                 #  ERROR (VIM indicates an ERROR status)
2480                                 #  ACTIVE,
2481                                 #  CREATING (on building process)
2482                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
2483                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
2484         """
2485         classification_dict = {}
2486         self.logger.debug("refresh_classifications status: Getting tenant classification information from VIM")
2487         for classification_id in classification_list:
2488             classification = {}
2489             try:
2490                 classification_vim = self.get_classification(classification_id)
2491                 if classification_vim:
2492                     classification['status'] = vmStatus2manoFormat['ACTIVE']
2493                 else:
2494                     classification['status'] = "OTHER"
2495                     classification['error_msg'] = "VIM status reported " + classification['status']
2496
2497                 classification['vim_info'] = self.serialize(classification_vim)
2498
2499                 if classification_vim.get('fault'):
2500                     classification['error_msg'] = str(classification_vim['fault'])
2501
2502             except vimconn.VimConnNotFoundException as e:
2503                 self.logger.error("Exception getting classification status: %s", str(e))
2504                 classification['status'] = "DELETED"
2505                 classification['error_msg'] = str(e)
2506             except vimconn.VimConnException as e:
2507                 self.logger.error("Exception getting classification status: %s", str(e))
2508                 classification['status'] = "VIM_ERROR"
2509                 classification['error_msg'] = str(e)
2510             classification_dict[classification_id] = classification
2511         return classification_dict