osm_ro/vimconn_openstack.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 osconnector implements all the methods to interact with openstack using the python-neutronclient.
  26
  27 For the VNF forwarding graph, The OpenStack VIM connector calls the
  28 networking-sfc Neutron extension methods, whose resources are mapped
  29 to the VIM connector's SFC resources as follows:
  30 - Classification (OSM) -> Flow Classifier (Neutron)
  31 - Service Function Instance (OSM) -> Port Pair (Neutron)
  32 - Service Function (OSM) -> Port Pair Group (Neutron)
  33 - Service Function Path (OSM) -> Port Chain (Neutron)
  34 '''
  35 __author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes, xFlow Research, Igor D.C., Eduardo Sousa"
  36 __date__  = "$22-sep-2017 23:59:59$"
  37
  38 import vimconn
  39 import json
  40 import logging
  41 import netaddr
  42 import time
  43 import yaml
  44 import random
  45 import re
  46 import copy
  47 from pprint import pformat
  48 from types import StringTypes
  49
  50 from novaclient import client as nClient, exceptions as nvExceptions
  51 from keystoneauth1.identity import v2, v3
  52 from keystoneauth1 import session
  53 import keystoneclient.exceptions as ksExceptions
  54 import keystoneclient.v3.client as ksClient_v3
  55 import keystoneclient.v2_0.client as ksClient_v2
  56 from glanceclient import client as glClient
  57 import glanceclient.exc as gl1Exceptions
  58 from  cinderclient import client as cClient
  59 from httplib import HTTPException
  60 from neutronclient.neutron import client as neClient
  61 from neutronclient.common import exceptions as neExceptions
  62 from requests.exceptions import ConnectionError
  63
  64
  65 """contain the openstack virtual machine status to openmano status"""
  66 vmStatus2manoFormat={'ACTIVE':'ACTIVE',
  67                      'PAUSED':'PAUSED',
  68                      'SUSPENDED': 'SUSPENDED',
  69                      'SHUTOFF':'INACTIVE',
  70                      'BUILD':'BUILD',
  71                      'ERROR':'ERROR','DELETED':'DELETED'
  72                      }
  73 netStatus2manoFormat={'ACTIVE':'ACTIVE','PAUSED':'PAUSED','INACTIVE':'INACTIVE','BUILD':'BUILD','ERROR':'ERROR','DELETED':'DELETED'
  74                      }
  75
  76 supportedClassificationTypes = ['legacy_flow_classifier']
  77
  78 #global var to have a timeout creating and deleting volumes
  79 volume_timeout = 600
  80 server_timeout = 600
  81
  82
  83 class SafeDumper(yaml.SafeDumper):
  84     def represent_data(self, data):
  85         # Openstack APIs use custom subclasses of dict and YAML safe dumper
  86         # is designed to not handle that (reference issue 142 of pyyaml)
  87         if isinstance(data, dict) and data.__class__ != dict:
  88             # A simple solution is to convert those items back to dicts
  89             data = dict(data.items())
  90
  91         return super(SafeDumper, self).represent_data(data)
  92
  93
  94 class vimconnector(vimconn.vimconnector):
  95     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None,
  96                  log_level=None, config={}, persistent_info={}):
  97         '''using common constructor parameters. In this case
  98         'url' is the keystone authorization url,
  99         'url_admin' is not use
 100         '''
 101         api_version = config.get('APIversion')
 102         if api_version and api_version not in ('v3.3', 'v2.0', '2', '3'):
 103             raise vimconn.vimconnException("Invalid value '{}' for config:APIversion. "
 104                                            "Allowed values are 'v3.3', 'v2.0', '2' or '3'".format(api_version))
 105         vim_type = config.get('vim_type')
 106         if vim_type and vim_type not in ('vio', 'VIO'):
 107             raise vimconn.vimconnException("Invalid value '{}' for config:vim_type."
 108                             "Allowed values are 'vio' or 'VIO'".format(vim_type))
 109
 110         if config.get('dataplane_net_vlan_range') is not None:
 111             #validate vlan ranges provided by user
 112             self._validate_vlan_ranges(config.get('dataplane_net_vlan_range'))
 113
 114         vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level,
 115                                       config)
 116
 117         if self.config.get("insecure") and self.config.get("ca_cert"):
 118             raise vimconn.vimconnException("options insecure and ca_cert are mutually exclusive")
 119         self.verify = True
 120         if self.config.get("insecure"):
 121             self.verify = False
 122         if self.config.get("ca_cert"):
 123             self.verify = self.config.get("ca_cert")
 124
 125         if not url:
 126             raise TypeError('url param can not be NoneType')
 127         self.persistent_info = persistent_info
 128         self.availability_zone = persistent_info.get('availability_zone', None)
 129         self.session = persistent_info.get('session', {'reload_client': True})
 130         self.nova = self.session.get('nova')
 131         self.neutron = self.session.get('neutron')
 132         self.cinder = self.session.get('cinder')
 133         self.glance = self.session.get('glance')
 134         # self.glancev1 = self.session.get('glancev1')
 135         self.keystone = self.session.get('keystone')
 136         self.api_version3 = self.session.get('api_version3')
 137         self.vim_type = self.config.get("vim_type")
 138         if self.vim_type:
 139             self.vim_type = self.vim_type.upper()
 140         if self.config.get("use_internal_endpoint"):
 141             self.endpoint_type = "internalURL"
 142         else:
 143             self.endpoint_type = None
 144
 145         self.logger = logging.getLogger('openmano.vim.openstack')
 146
 147         ####### VIO Specific Changes #########
 148         if self.vim_type == "VIO":
 149             self.logger = logging.getLogger('openmano.vim.vio')
 150
 151         if log_level:
 152             self.logger.setLevel( getattr(logging, log_level))
 153
 154     def __getitem__(self, index):
 155         """Get individuals parameters.
 156         Throw KeyError"""
 157         if index == 'project_domain_id':
 158             return self.config.get("project_domain_id")
 159         elif index == 'user_domain_id':
 160             return self.config.get("user_domain_id")
 161         else:
 162             return vimconn.vimconnector.__getitem__(self, index)
 163
 164     def __setitem__(self, index, value):
 165         """Set individuals parameters and it is marked as dirty so to force connection reload.
 166         Throw KeyError"""
 167         if index == 'project_domain_id':
 168             self.config["project_domain_id"] = value
 169         elif index == 'user_domain_id':
 170                 self.config["user_domain_id"] = value
 171         else:
 172             vimconn.vimconnector.__setitem__(self, index, value)
 173         self.session['reload_client'] = True
 174
 175     def serialize(self, value):
 176         """Serialization of python basic types.
 177
 178         In the case value is not serializable a message will be logged and a
 179         simple representation of the data that cannot be converted back to
 180         python is returned.
 181         """
 182         if isinstance(value, StringTypes):
 183             return value
 184
 185         try:
 186             return yaml.dump(value, Dumper=SafeDumper,
 187                              default_flow_style=True, width=256)
 188         except yaml.representer.RepresenterError:
 189                 self.logger.debug(
 190                     'The following entity cannot be serialized in YAML:'
 191                     '\n\n%s\n\n', pformat(value), exc_info=True)
 192                 return str(value)
 193
 194     def _reload_connection(self):
 195         '''Called before any operation, it check if credentials has changed
 196         Throw keystoneclient.apiclient.exceptions.AuthorizationFailure
 197         '''
 198         #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-)
 199         if self.session['reload_client']:
 200             if self.config.get('APIversion'):
 201                 self.api_version3 = self.config['APIversion'] == 'v3.3' or self.config['APIversion'] == '3'
 202             else:  # get from ending auth_url that end with v3 or with v2.0
 203                 self.api_version3 =  self.url.endswith("/v3") or self.url.endswith("/v3/")
 204             self.session['api_version3'] = self.api_version3
 205             if self.api_version3:
 206                 if self.config.get('project_domain_id') or self.config.get('project_domain_name'):
 207                     project_domain_id_default = None
 208                 else:
 209                     project_domain_id_default = 'default'
 210                 if self.config.get('user_domain_id') or self.config.get('user_domain_name'):
 211                     user_domain_id_default = None
 212                 else:
 213                     user_domain_id_default = 'default'
 214                 auth = v3.Password(auth_url=self.url,
 215                                    username=self.user,
 216                                    password=self.passwd,
 217                                    project_name=self.tenant_name,
 218                                    project_id=self.tenant_id,
 219                                    project_domain_id=self.config.get('project_domain_id', project_domain_id_default),
 220                                    user_domain_id=self.config.get('user_domain_id', user_domain_id_default),
 221                                    project_domain_name=self.config.get('project_domain_name'),
 222                                    user_domain_name=self.config.get('user_domain_name'))
 223             else:
 224                 auth = v2.Password(auth_url=self.url,
 225                                    username=self.user,
 226                                    password=self.passwd,
 227                                    tenant_name=self.tenant_name,
 228                                    tenant_id=self.tenant_id)
 229             sess = session.Session(auth=auth, verify=self.verify)
 230             if self.api_version3:
 231                 self.keystone = ksClient_v3.Client(session=sess, endpoint_type=self.endpoint_type)
 232             else:
 233                 self.keystone = ksClient_v2.Client(session=sess, endpoint_type=self.endpoint_type)
 234             self.session['keystone'] = self.keystone
 235             # In order to enable microversion functionality an explicit microversion must be specified in 'config'.
 236             # This implementation approach is due to the warning message in
 237             # https://developer.openstack.org/api-guide/compute/microversions.html
 238             # where it is stated that microversion backwards compatibility is not guaranteed and clients should
 239             # always require an specific microversion.
 240             # To be able to use 'device role tagging' functionality define 'microversion: 2.32' in datacenter config
 241             version = self.config.get("microversion")
 242             if not version:
 243                 version = "2.1"
 244             self.nova = self.session['nova'] = nClient.Client(str(version), session=sess, endpoint_type=self.endpoint_type)
 245             self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess, endpoint_type=self.endpoint_type)
 246             self.cinder = self.session['cinder'] = cClient.Client(2, session=sess, endpoint_type=self.endpoint_type)
 247             if self.endpoint_type == "internalURL":
 248                 glance_service_id = self.keystone.services.list(name="glance")[0].id
 249                 glance_endpoint = self.keystone.endpoints.list(glance_service_id, interface="internal")[0].url
 250             else:
 251                 glance_endpoint = None
 252             self.glance = self.session['glance'] = glClient.Client(2, session=sess, endpoint=glance_endpoint)
 253             #using version 1 of glance client in new_image()
 254             # self.glancev1 = self.session['glancev1'] = glClient.Client('1', session=sess,
 255             #                                                            endpoint=glance_endpoint)
 256             self.session['reload_client'] = False
 257             self.persistent_info['session'] = self.session
 258             # add availablity zone info inside  self.persistent_info
 259             self._set_availablity_zones()
 260             self.persistent_info['availability_zone'] = self.availability_zone
 261
 262     def __net_os2mano(self, net_list_dict):
 263         '''Transform the net openstack format to mano format
 264         net_list_dict can be a list of dict or a single dict'''
 265         if type(net_list_dict) is dict:
 266             net_list_=(net_list_dict,)
 267         elif type(net_list_dict) is list:
 268             net_list_=net_list_dict
 269         else:
 270             raise TypeError("param net_list_dict must be a list or a dictionary")
 271         for net in net_list_:
 272             if net.get('provider:network_type') == "vlan":
 273                 net['type']='data'
 274             else:
 275                 net['type']='bridge'
 276
 277     def __classification_os2mano(self, class_list_dict):
 278         """Transform the openstack format (Flow Classifier) to mano format
 279         (Classification) class_list_dict can be a list of dict or a single dict
 280         """
 281         if isinstance(class_list_dict, dict):
 282             class_list_ = [class_list_dict]
 283         elif isinstance(class_list_dict, list):
 284             class_list_ = class_list_dict
 285         else:
 286             raise TypeError(
 287                 "param class_list_dict must be a list or a dictionary")
 288         for classification in class_list_:
 289             id = classification.pop('id')
 290             name = classification.pop('name')
 291             description = classification.pop('description')
 292             project_id = classification.pop('project_id')
 293             tenant_id = classification.pop('tenant_id')
 294             original_classification = copy.deepcopy(classification)
 295             classification.clear()
 296             classification['ctype'] = 'legacy_flow_classifier'
 297             classification['definition'] = original_classification
 298             classification['id'] = id
 299             classification['name'] = name
 300             classification['description'] = description
 301             classification['project_id'] = project_id
 302             classification['tenant_id'] = tenant_id
 303
 304     def __sfi_os2mano(self, sfi_list_dict):
 305         """Transform the openstack format (Port Pair) to mano format (SFI)
 306         sfi_list_dict can be a list of dict or a single dict
 307         """
 308         if isinstance(sfi_list_dict, dict):
 309             sfi_list_ = [sfi_list_dict]
 310         elif isinstance(sfi_list_dict, list):
 311             sfi_list_ = sfi_list_dict
 312         else:
 313             raise TypeError(
 314                 "param sfi_list_dict must be a list or a dictionary")
 315         for sfi in sfi_list_:
 316             sfi['ingress_ports'] = []
 317             sfi['egress_ports'] = []
 318             if sfi.get('ingress'):
 319                 sfi['ingress_ports'].append(sfi['ingress'])
 320             if sfi.get('egress'):
 321                 sfi['egress_ports'].append(sfi['egress'])
 322             del sfi['ingress']
 323             del sfi['egress']
 324             params = sfi.get('service_function_parameters')
 325             sfc_encap = False
 326             if params:
 327                 correlation = params.get('correlation')
 328                 if correlation:
 329                     sfc_encap = True
 330             sfi['sfc_encap'] = sfc_encap
 331             del sfi['service_function_parameters']
 332
 333     def __sf_os2mano(self, sf_list_dict):
 334         """Transform the openstack format (Port Pair Group) to mano format (SF)
 335         sf_list_dict can be a list of dict or a single dict
 336         """
 337         if isinstance(sf_list_dict, dict):
 338             sf_list_ = [sf_list_dict]
 339         elif isinstance(sf_list_dict, list):
 340             sf_list_ = sf_list_dict
 341         else:
 342             raise TypeError(
 343                 "param sf_list_dict must be a list or a dictionary")
 344         for sf in sf_list_:
 345             del sf['port_pair_group_parameters']
 346             sf['sfis'] = sf['port_pairs']
 347             del sf['port_pairs']
 348
 349     def __sfp_os2mano(self, sfp_list_dict):
 350         """Transform the openstack format (Port Chain) to mano format (SFP)
 351         sfp_list_dict can be a list of dict or a single dict
 352         """
 353         if isinstance(sfp_list_dict, dict):
 354             sfp_list_ = [sfp_list_dict]
 355         elif isinstance(sfp_list_dict, list):
 356             sfp_list_ = sfp_list_dict
 357         else:
 358             raise TypeError(
 359                 "param sfp_list_dict must be a list or a dictionary")
 360         for sfp in sfp_list_:
 361             params = sfp.pop('chain_parameters')
 362             sfc_encap = False
 363             if params:
 364                 correlation = params.get('correlation')
 365                 if correlation:
 366                     sfc_encap = True
 367             sfp['sfc_encap'] = sfc_encap
 368             sfp['spi'] = sfp.pop('chain_id')
 369             sfp['classifications'] = sfp.pop('flow_classifiers')
 370             sfp['service_functions'] = sfp.pop('port_pair_groups')
 371
 372     # placeholder for now; read TODO note below
 373     def _validate_classification(self, type, definition):
 374         # only legacy_flow_classifier Type is supported at this point
 375         return True
 376         # TODO(igordcard): this method should be an abstract method of an
 377         # abstract Classification class to be implemented by the specific
 378         # Types. Also, abstract vimconnector should call the validation
 379         # method before the implemented VIM connectors are called.
 380
 381     def _format_exception(self, exception):
 382         '''Transform a keystone, nova, neutron  exception into a vimconn exception'''
 383         if isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError,
 384                                   ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed
 385                                   )):
 386             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 387         elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException,
 388                                     neExceptions.NeutronException, nvExceptions.BadRequest)):
 389             raise vimconn.vimconnUnexpectedResponse(type(exception).__name__ + ": " + str(exception))
 390         elif isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound)):
 391             raise vimconn.vimconnNotFoundException(type(exception).__name__ + ": " + str(exception))
 392         elif isinstance(exception, nvExceptions.Conflict):
 393             raise vimconn.vimconnConflictException(type(exception).__name__ + ": " + str(exception))
 394         elif isinstance(exception, vimconn.vimconnException):
 395             raise exception
 396         else:  # ()
 397             self.logger.error("General Exception " + str(exception), exc_info=True)
 398             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 399
 400     def get_tenant_list(self, filter_dict={}):
 401         '''Obtain tenants of VIM
 402         filter_dict can contain the following keys:
 403             name: filter by tenant name
 404             id: filter by tenant uuid/id
 405             <other VIM specific>
 406         Returns the tenant list of dictionaries: [{'name':'<name>, 'id':'<id>, ...}, ...]
 407         '''
 408         self.logger.debug("Getting tenants from VIM filter: '%s'", str(filter_dict))
 409         try:
 410             self._reload_connection()
 411             if self.api_version3:
 412                 project_class_list = self.keystone.projects.list(name=filter_dict.get("name"))
 413             else:
 414                 project_class_list = self.keystone.tenants.findall(**filter_dict)
 415             project_list=[]
 416             for project in project_class_list:
 417                 if filter_dict.get('id') and filter_dict["id"] != project.id:
 418                     continue
 419                 project_list.append(project.to_dict())
 420             return project_list
 421         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError) as e:
 422             self._format_exception(e)
 423
 424     def new_tenant(self, tenant_name, tenant_description):
 425         '''Adds a new tenant to openstack VIM. Returns the tenant identifier'''
 426         self.logger.debug("Adding a new tenant name: %s", tenant_name)
 427         try:
 428             self._reload_connection()
 429             if self.api_version3:
 430                 project = self.keystone.projects.create(tenant_name, self.config.get("project_domain_id", "default"),
 431                                                         description=tenant_description, is_domain=False)
 432             else:
 433                 project = self.keystone.tenants.create(tenant_name, tenant_description)
 434             return project.id
 435         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 436             self._format_exception(e)
 437
 438     def delete_tenant(self, tenant_id):
 439         '''Delete a tenant from openstack VIM. Returns the old tenant identifier'''
 440         self.logger.debug("Deleting tenant %s from VIM", tenant_id)
 441         try:
 442             self._reload_connection()
 443             if self.api_version3:
 444                 self.keystone.projects.delete(tenant_id)
 445             else:
 446                 self.keystone.tenants.delete(tenant_id)
 447             return tenant_id
 448         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 449             self._format_exception(e)
 450
 451     def new_network(self,net_name, net_type, ip_profile=None, shared=False, vlan=None):
 452         '''Adds a tenant network to VIM. Returns the network identifier'''
 453         self.logger.debug("Adding a new network to VIM name '%s', type '%s'", net_name, net_type)
 454         #self.logger.debug(">>>>>>>>>>>>>>>>>> IP profile %s", str(ip_profile))
 455         try:
 456             new_net = None
 457             self._reload_connection()
 458             network_dict = {'name': net_name, 'admin_state_up': True}
 459             if net_type=="data" or net_type=="ptp":
 460                 if self.config.get('dataplane_physical_net') == None:
 461                     raise vimconn.vimconnConflictException("You must provide a 'dataplane_physical_net' at config value before creating sriov network")
 462                 network_dict["provider:physical_network"] = self.config['dataplane_physical_net'] #"physnet_sriov" #TODO physical
 463                 network_dict["provider:network_type"]     = "vlan"
 464                 if vlan!=None:
 465                     network_dict["provider:network_type"] = vlan
 466
 467                 ####### VIO Specific Changes #########
 468                 if self.vim_type == "VIO":
 469                     if vlan is not None:
 470                         network_dict["provider:segmentation_id"] = vlan
 471                     else:
 472                         if self.config.get('dataplane_net_vlan_range') is None:
 473                             raise vimconn.vimconnConflictException("You must provide "\
 474                                 "'dataplane_net_vlan_range' in format [start_ID - end_ID]"\
 475                                 "at config value before creating sriov network with vlan tag")
 476
 477                         network_dict["provider:segmentation_id"] = self._genrate_vlanID()
 478
 479             network_dict["shared"]=shared
 480             new_net=self.neutron.create_network({'network':network_dict})
 481             #print new_net
 482             #create subnetwork, even if there is no profile
 483             if not ip_profile:
 484                 ip_profile = {}
 485             if not ip_profile.get('subnet_address'):
 486                 #Fake subnet is required
 487                 subnet_rand = random.randint(0, 255)
 488                 ip_profile['subnet_address'] = "192.168.{}.0/24".format(subnet_rand)
 489             if 'ip_version' not in ip_profile:
 490                 ip_profile['ip_version'] = "IPv4"
 491             subnet = {"name":net_name+"-subnet",
 492                     "network_id": new_net["network"]["id"],
 493                     "ip_version": 4 if ip_profile['ip_version']=="IPv4" else 6,
 494                     "cidr": ip_profile['subnet_address']
 495                     }
 496             # Gateway should be set to None if not needed. Otherwise openstack assigns one by default
 497             if ip_profile.get('gateway_address'):
 498                 subnet['gateway_ip'] = ip_profile['gateway_address']
 499             else:
 500                 subnet['gateway_ip'] = None
 501             if ip_profile.get('dns_address'):
 502                 subnet['dns_nameservers'] = ip_profile['dns_address'].split(";")
 503             if 'dhcp_enabled' in ip_profile:
 504                 subnet['enable_dhcp'] = False if \
 505                     ip_profile['dhcp_enabled']=="false" or ip_profile['dhcp_enabled']==False else True
 506             if ip_profile.get('dhcp_start_address'):
 507                 subnet['allocation_pools'] = []
 508                 subnet['allocation_pools'].append(dict())
 509                 subnet['allocation_pools'][0]['start'] = ip_profile['dhcp_start_address']
 510             if ip_profile.get('dhcp_count'):
 511                 #parts = ip_profile['dhcp_start_address'].split('.')
 512                 #ip_int = (int(parts[0]) << 24) + (int(parts[1]) << 16) + (int(parts[2]) << 8) + int(parts[3])
 513                 ip_int = int(netaddr.IPAddress(ip_profile['dhcp_start_address']))
 514                 ip_int += ip_profile['dhcp_count'] - 1
 515                 ip_str = str(netaddr.IPAddress(ip_int))
 516                 subnet['allocation_pools'][0]['end'] = ip_str
 517             #self.logger.debug(">>>>>>>>>>>>>>>>>> Subnet: %s", str(subnet))
 518             self.neutron.create_subnet({"subnet": subnet} )
 519             return new_net["network"]["id"]
 520         except Exception as e:
 521             if new_net:
 522                 self.neutron.delete_network(new_net['network']['id'])
 523             self._format_exception(e)
 524
 525     def get_network_list(self, filter_dict={}):
 526         '''Obtain tenant networks of VIM
 527         Filter_dict can be:
 528             name: network name
 529             id: network uuid
 530             shared: boolean
 531             tenant_id: tenant
 532             admin_state_up: boolean
 533             status: 'ACTIVE'
 534         Returns the network list of dictionaries
 535         '''
 536         self.logger.debug("Getting network from VIM filter: '%s'", str(filter_dict))
 537         try:
 538             self._reload_connection()
 539             filter_dict_os = filter_dict.copy()
 540             if self.api_version3 and "tenant_id" in filter_dict_os:
 541                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')  #T ODO check
 542             net_dict = self.neutron.list_networks(**filter_dict_os)
 543             net_list = net_dict["networks"]
 544             self.__net_os2mano(net_list)
 545             return net_list
 546         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 547             self._format_exception(e)
 548
 549     def get_network(self, net_id):
 550         '''Obtain details of network from VIM
 551         Returns the network information from a network id'''
 552         self.logger.debug(" Getting tenant network %s from VIM", net_id)
 553         filter_dict={"id": net_id}
 554         net_list = self.get_network_list(filter_dict)
 555         if len(net_list)==0:
 556             raise vimconn.vimconnNotFoundException("Network '{}' not found".format(net_id))
 557         elif len(net_list)>1:
 558             raise vimconn.vimconnConflictException("Found more than one network with this criteria")
 559         net = net_list[0]
 560         subnets=[]
 561         for subnet_id in net.get("subnets", () ):
 562             try:
 563                 subnet = self.neutron.show_subnet(subnet_id)
 564             except Exception as e:
 565                 self.logger.error("osconnector.get_network(): Error getting subnet %s %s" % (net_id, str(e)))
 566                 subnet = {"id": subnet_id, "fault": str(e)}
 567             subnets.append(subnet)
 568         net["subnets"] = subnets
 569         net["encapsulation"] = net.get('provider:network_type')
 570         net["segmentation_id"] = net.get('provider:segmentation_id')
 571         return net
 572
 573     def delete_network(self, net_id):
 574         '''Deletes a tenant network from VIM. Returns the old network identifier'''
 575         self.logger.debug("Deleting network '%s' from VIM", net_id)
 576         try:
 577             self._reload_connection()
 578             #delete VM ports attached to this networks before the network
 579             ports = self.neutron.list_ports(network_id=net_id)
 580             for p in ports['ports']:
 581                 try:
 582                     self.neutron.delete_port(p["id"])
 583                 except Exception as e:
 584                     self.logger.error("Error deleting port %s: %s", p["id"], str(e))
 585             self.neutron.delete_network(net_id)
 586             return net_id
 587         except (neExceptions.ConnectionFailed, neExceptions.NetworkNotFoundClient, neExceptions.NeutronException,
 588                 ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 589             self._format_exception(e)
 590
 591     def refresh_nets_status(self, net_list):
 592         '''Get the status of the networks
 593            Params: the list of network identifiers
 594            Returns a dictionary with:
 595                 net_id:         #VIM id of this network
 596                     status:     #Mandatory. Text with one of:
 597                                 #  DELETED (not found at vim)
 598                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 599                                 #  OTHER (Vim reported other status not understood)
 600                                 #  ERROR (VIM indicates an ERROR status)
 601                                 #  ACTIVE, INACTIVE, DOWN (admin down),
 602                                 #  BUILD (on building process)
 603                                 #
 604                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 605                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 606
 607         '''
 608         net_dict={}
 609         for net_id in net_list:
 610             net = {}
 611             try:
 612                 net_vim = self.get_network(net_id)
 613                 if net_vim['status'] in netStatus2manoFormat:
 614                     net["status"] = netStatus2manoFormat[ net_vim['status'] ]
 615                 else:
 616                     net["status"] = "OTHER"
 617                     net["error_msg"] = "VIM status reported " + net_vim['status']
 618
 619                 if net['status'] == "ACTIVE" and not net_vim['admin_state_up']:
 620                     net['status'] = 'DOWN'
 621
 622                 net['vim_info'] = self.serialize(net_vim)
 623
 624                 if net_vim.get('fault'):  #TODO
 625                     net['error_msg'] = str(net_vim['fault'])
 626             except vimconn.vimconnNotFoundException as e:
 627                 self.logger.error("Exception getting net status: %s", str(e))
 628                 net['status'] = "DELETED"
 629                 net['error_msg'] = str(e)
 630             except vimconn.vimconnException as e:
 631                 self.logger.error("Exception getting net status: %s", str(e))
 632                 net['status'] = "VIM_ERROR"
 633                 net['error_msg'] = str(e)
 634             net_dict[net_id] = net
 635         return net_dict
 636
 637     def get_flavor(self, flavor_id):
 638         '''Obtain flavor details from the  VIM. Returns the flavor dict details'''
 639         self.logger.debug("Getting flavor '%s'", flavor_id)
 640         try:
 641             self._reload_connection()
 642             flavor = self.nova.flavors.find(id=flavor_id)
 643             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 644             return flavor.to_dict()
 645         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 646             self._format_exception(e)
 647
 648     def get_flavor_id_from_data(self, flavor_dict):
 649         """Obtain flavor id that match the flavor description
 650            Returns the flavor_id or raises a vimconnNotFoundException
 651            flavor_dict: contains the required ram, vcpus, disk
 652            If 'use_existing_flavors' is set to True at config, the closer flavor that provides same or more ram, vcpus
 653                 and disk is returned. Otherwise a flavor with exactly same ram, vcpus and disk is returned or a
 654                 vimconnNotFoundException is raised
 655         """
 656         exact_match = False if self.config.get('use_existing_flavors') else True
 657         try:
 658             self._reload_connection()
 659             flavor_candidate_id = None
 660             flavor_candidate_data = (10000, 10000, 10000)
 661             flavor_target = (flavor_dict["ram"], flavor_dict["vcpus"], flavor_dict["disk"])
 662             # numa=None
 663             numas = flavor_dict.get("extended", {}).get("numas")
 664             if numas:
 665                 #TODO
 666                 raise vimconn.vimconnNotFoundException("Flavor with EPA still not implemted")
 667                 # if len(numas) > 1:
 668                 #     raise vimconn.vimconnNotFoundException("Cannot find any flavor with more than one numa")
 669                 # numa=numas[0]
 670                 # numas = extended.get("numas")
 671             for flavor in self.nova.flavors.list():
 672                 epa = flavor.get_keys()
 673                 if epa:
 674                     continue
 675                     # TODO
 676                 flavor_data = (flavor.ram, flavor.vcpus, flavor.disk)
 677                 if flavor_data == flavor_target:
 678                     return flavor.id
 679                 elif not exact_match and flavor_target < flavor_data < flavor_candidate_data:
 680                     flavor_candidate_id = flavor.id
 681                     flavor_candidate_data = flavor_data
 682             if not exact_match and flavor_candidate_id:
 683                 return flavor_candidate_id
 684             raise vimconn.vimconnNotFoundException("Cannot find any flavor matching '{}'".format(str(flavor_dict)))
 685         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 686             self._format_exception(e)
 687
 688     def new_flavor(self, flavor_data, change_name_if_used=True):
 689         '''Adds a tenant flavor to openstack VIM
 690         if change_name_if_used is True, it will change name in case of conflict, because it is not supported name repetition
 691         Returns the flavor identifier
 692         '''
 693         self.logger.debug("Adding flavor '%s'", str(flavor_data))
 694         retry=0
 695         max_retries=3
 696         name_suffix = 0
 697         name=flavor_data['name']
 698         while retry<max_retries:
 699             retry+=1
 700             try:
 701                 self._reload_connection()
 702                 if change_name_if_used:
 703                     #get used names
 704                     fl_names=[]
 705                     fl=self.nova.flavors.list()
 706                     for f in fl:
 707                         fl_names.append(f.name)
 708                     while name in fl_names:
 709                         name_suffix += 1
 710                         name = flavor_data['name']+"-" + str(name_suffix)
 711
 712                 ram = flavor_data.get('ram',64)
 713                 vcpus = flavor_data.get('vcpus',1)
 714                 numa_properties=None
 715
 716                 extended = flavor_data.get("extended")
 717                 if extended:
 718                     numas=extended.get("numas")
 719                     if numas:
 720                         numa_nodes = len(numas)
 721                         if numa_nodes > 1:
 722                             return -1, "Can not add flavor with more than one numa"
 723                         numa_properties = {"hw:numa_nodes":str(numa_nodes)}
 724                         numa_properties["hw:mem_page_size"] = "large"
 725                         numa_properties["hw:cpu_policy"] = "dedicated"
 726                         numa_properties["hw:numa_mempolicy"] = "strict"
 727                         if self.vim_type == "VIO":
 728                             numa_properties["vmware:extra_config"] = '{"numa.nodeAffinity":"0"}'
 729                             numa_properties["vmware:latency_sensitivity_level"] = "high"
 730                         for numa in numas:
 731                             #overwrite ram and vcpus
 732                             #check if key 'memory' is present in numa else use ram value at flavor
 733                             if 'memory' in numa:
 734                                 ram = numa['memory']*1024
 735                             #See for reference: https://specs.openstack.org/openstack/nova-specs/specs/mitaka/implemented/virt-driver-cpu-thread-pinning.html
 736                             if 'paired-threads' in numa:
 737                                 vcpus = numa['paired-threads']*2
 738                                 #cpu_thread_policy "require" implies that the compute node must have an STM architecture
 739                                 numa_properties["hw:cpu_thread_policy"] = "require"
 740                                 numa_properties["hw:cpu_policy"] = "dedicated"
 741                             elif 'cores' in numa:
 742                                 vcpus = numa['cores']
 743                                 # cpu_thread_policy "prefer" implies that the host must not have an SMT architecture, or a non-SMT architecture will be emulated
 744                                 numa_properties["hw:cpu_thread_policy"] = "isolate"
 745                                 numa_properties["hw:cpu_policy"] = "dedicated"
 746                             elif 'threads' in numa:
 747                                 vcpus = numa['threads']
 748                                 # cpu_thread_policy "prefer" implies that the host may or may not have an SMT architecture
 749                                 numa_properties["hw:cpu_thread_policy"] = "prefer"
 750                                 numa_properties["hw:cpu_policy"] = "dedicated"
 751                             # for interface in numa.get("interfaces",() ):
 752                             #     if interface["dedicated"]=="yes":
 753                             #         raise vimconn.vimconnException("Passthrough interfaces are not supported for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable)
 754                             #     #TODO, add the key 'pci_passthrough:alias"="<label at config>:<number ifaces>"' when a way to connect it is available
 755
 756                 #create flavor
 757                 new_flavor=self.nova.flavors.create(name,
 758                                 ram,
 759                                 vcpus,
 760                                 flavor_data.get('disk',0),
 761                                 is_public=flavor_data.get('is_public', True)
 762                             )
 763                 #add metadata
 764                 if numa_properties:
 765                     new_flavor.set_keys(numa_properties)
 766                 return new_flavor.id
 767             except nvExceptions.Conflict as e:
 768                 if change_name_if_used and retry < max_retries:
 769                     continue
 770                 self._format_exception(e)
 771             #except nvExceptions.BadRequest as e:
 772             except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 773                 self._format_exception(e)
 774
 775     def delete_flavor(self,flavor_id):
 776         '''Deletes a tenant flavor from openstack VIM. Returns the old flavor_id
 777         '''
 778         try:
 779             self._reload_connection()
 780             self.nova.flavors.delete(flavor_id)
 781             return flavor_id
 782         #except nvExceptions.BadRequest as e:
 783         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 784             self._format_exception(e)
 785
 786     def new_image(self,image_dict):
 787         '''
 788         Adds a tenant image to VIM. imge_dict is a dictionary with:
 789             name: name
 790             disk_format: qcow2, vhd, vmdk, raw (by default), ...
 791             location: path or URI
 792             public: "yes" or "no"
 793             metadata: metadata of the image
 794         Returns the image_id
 795         '''
 796         retry=0
 797         max_retries=3
 798         while retry<max_retries:
 799             retry+=1
 800             try:
 801                 self._reload_connection()
 802                 #determine format  http://docs.openstack.org/developer/glance/formats.html
 803                 if "disk_format" in image_dict:
 804                     disk_format=image_dict["disk_format"]
 805                 else: #autodiscover based on extension
 806                     if image_dict['location'].endswith(".qcow2"):
 807                         disk_format="qcow2"
 808                     elif image_dict['location'].endswith(".vhd"):
 809                         disk_format="vhd"
 810                     elif image_dict['location'].endswith(".vmdk"):
 811                         disk_format="vmdk"
 812                     elif image_dict['location'].endswith(".vdi"):
 813                         disk_format="vdi"
 814                     elif image_dict['location'].endswith(".iso"):
 815                         disk_format="iso"
 816                     elif image_dict['location'].endswith(".aki"):
 817                         disk_format="aki"
 818                     elif image_dict['location'].endswith(".ari"):
 819                         disk_format="ari"
 820                     elif image_dict['location'].endswith(".ami"):
 821                         disk_format="ami"
 822                     else:
 823                         disk_format="raw"
 824                 self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location'])
 825                 new_image = self.glance.images.create(name=image_dict['name'])
 826                 if image_dict['location'].startswith("http"):
 827                     # TODO there is not a method to direct download. It must be downloaded locally with requests
 828                     raise vimconn.vimconnNotImplemented("Cannot create image from URL")
 829                 else: #local path
 830                     with open(image_dict['location']) as fimage:
 831                         self.glance.images.upload(new_image.id, fimage)
 832                         #new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 833                         #    container_format="bare", data=fimage, disk_format=disk_format)
 834                 metadata_to_load = image_dict.get('metadata')
 835                 #TODO location is a reserved word for current openstack versions. Use another word
 836                 metadata_to_load['location'] = image_dict['location']
 837                 self.glance.images.update(new_image.id, **metadata_to_load)
 838                 return new_image.id
 839             except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e:
 840                 self._format_exception(e)
 841             except (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 842                 if retry==max_retries:
 843                     continue
 844                 self._format_exception(e)
 845             except IOError as e:  #can not open the file
 846                 raise vimconn.vimconnConnectionException(type(e).__name__ + ": " + str(e)+ " for " + image_dict['location'],
 847                                                          http_code=vimconn.HTTP_Bad_Request)
 848
 849     def delete_image(self, image_id):
 850         '''Deletes a tenant image from openstack VIM. Returns the old id
 851         '''
 852         try:
 853             self._reload_connection()
 854             self.glance.images.delete(image_id)
 855             return image_id
 856         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e: #TODO remove
 857             self._format_exception(e)
 858
 859     def get_image_id_from_path(self, path):
 860         '''Get the image id from image path in the VIM database. Returns the image_id'''
 861         try:
 862             self._reload_connection()
 863             images = self.glance.images.list()
 864             for image in images:
 865                 if image.metadata.get("location")==path:
 866                     return image.id
 867             raise vimconn.vimconnNotFoundException("image with location '{}' not found".format( path))
 868         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 869             self._format_exception(e)
 870
 871     def get_image_list(self, filter_dict={}):
 872         '''Obtain tenant images from VIM
 873         Filter_dict can be:
 874             id: image id
 875             name: image name
 876             checksum: image checksum
 877         Returns the image list of dictionaries:
 878             [{<the fields at Filter_dict plus some VIM specific>}, ...]
 879             List can be empty
 880         '''
 881         self.logger.debug("Getting image list from VIM filter: '%s'", str(filter_dict))
 882         try:
 883             self._reload_connection()
 884             filter_dict_os = filter_dict.copy()
 885             #First we filter by the available filter fields: name, id. The others are removed.
 886             image_list = self.glance.images.list()
 887             filtered_list = []
 888             for image in image_list:
 889                 try:
 890                     if filter_dict.get("name") and image["name"] != filter_dict["name"]:
 891                         continue
 892                     if filter_dict.get("id") and image["id"] != filter_dict["id"]:
 893                         continue
 894                     if filter_dict.get("checksum") and image["checksum"] != filter_dict["checksum"]:
 895                         continue
 896
 897                     filtered_list.append(image.copy())
 898                 except gl1Exceptions.HTTPNotFound:
 899                     pass
 900             return filtered_list
 901         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 902             self._format_exception(e)
 903
 904     def __wait_for_vm(self, vm_id, status):
 905         """wait until vm is in the desired status and return True.
 906         If the VM gets in ERROR status, return false.
 907         If the timeout is reached generate an exception"""
 908         elapsed_time = 0
 909         while elapsed_time < server_timeout:
 910             vm_status = self.nova.servers.get(vm_id).status
 911             if vm_status == status:
 912                 return True
 913             if vm_status == 'ERROR':
 914                 return False
 915             time.sleep(5)
 916             elapsed_time += 5
 917
 918         # if we exceeded the timeout rollback
 919         if elapsed_time >= server_timeout:
 920             raise vimconn.vimconnException('Timeout waiting for instance ' + vm_id + ' to get ' + status,
 921                                            http_code=vimconn.HTTP_Request_Timeout)
 922
 923     def _get_openstack_availablity_zones(self):
 924         """
 925         Get from openstack availability zones available
 926         :return:
 927         """
 928         try:
 929             openstack_availability_zone = self.nova.availability_zones.list()
 930             openstack_availability_zone = [str(zone.zoneName) for zone in openstack_availability_zone
 931                                            if zone.zoneName != 'internal']
 932             return openstack_availability_zone
 933         except Exception as e:
 934             return None
 935
 936     def _set_availablity_zones(self):
 937         """
 938         Set vim availablity zone
 939         :return:
 940         """
 941
 942         if 'availability_zone' in self.config:
 943             vim_availability_zones = self.config.get('availability_zone')
 944             if isinstance(vim_availability_zones, str):
 945                 self.availability_zone = [vim_availability_zones]
 946             elif isinstance(vim_availability_zones, list):
 947                 self.availability_zone = vim_availability_zones
 948         else:
 949             self.availability_zone = self._get_openstack_availablity_zones()
 950
 951     def _get_vm_availability_zone(self, availability_zone_index, availability_zone_list):
 952         """
 953         Return thge availability zone to be used by the created VM.
 954         :return: The VIM availability zone to be used or None
 955         """
 956         if availability_zone_index is None:
 957             if not self.config.get('availability_zone'):
 958                 return None
 959             elif isinstance(self.config.get('availability_zone'), str):
 960                 return self.config['availability_zone']
 961             else:
 962                 # TODO consider using a different parameter at config for default AV and AV list match
 963                 return self.config['availability_zone'][0]
 964
 965         vim_availability_zones = self.availability_zone
 966         # check if VIM offer enough availability zones describe in the VNFD
 967         if vim_availability_zones and len(availability_zone_list) <= len(vim_availability_zones):
 968             # check if all the names of NFV AV match VIM AV names
 969             match_by_index = False
 970             for av in availability_zone_list:
 971                 if av not in vim_availability_zones:
 972                     match_by_index = True
 973                     break
 974             if match_by_index:
 975                 return vim_availability_zones[availability_zone_index]
 976             else:
 977                 return availability_zone_list[availability_zone_index]
 978         else:
 979             raise vimconn.vimconnConflictException("No enough availability zones at VIM for this deployment")
 980
 981     def new_vminstance(self, name, description, start, image_id, flavor_id, net_list, cloud_config=None, disk_list=None,
 982                        availability_zone_index=None, availability_zone_list=None):
 983         """Adds a VM instance to VIM
 984         Params:
 985             start: indicates if VM must start or boot in pause mode. Ignored
 986             image_id,flavor_id: iamge and flavor uuid
 987             net_list: list of interfaces, each one is a dictionary with:
 988                 name:
 989                 net_id: network uuid to connect
 990                 vpci: virtual vcpi to assign, ignored because openstack lack #TODO
 991                 model: interface model, ignored #TODO
 992                 mac_address: used for  SR-IOV ifaces #TODO for other types
 993                 use: 'data', 'bridge',  'mgmt'
 994                 type: 'virtual', 'PCI-PASSTHROUGH'('PF'), 'SR-IOV'('VF'), 'VFnotShared'
 995                 vim_id: filled/added by this function
 996                 floating_ip: True/False (or it can be None)
 997             'cloud_config': (optional) dictionary with:
 998             'key-pairs': (optional) list of strings with the public key to be inserted to the default user
 999             'users': (optional) list of users to be inserted, each item is a dict with:
1000                 'name': (mandatory) user name,
1001                 'key-pairs': (optional) list of strings with the public key to be inserted to the user
1002             'user-data': (optional) string is a text script to be passed directly to cloud-init
1003             'config-files': (optional). List of files to be transferred. Each item is a dict with:
1004                 'dest': (mandatory) string with the destination absolute path
1005                 'encoding': (optional, by default text). Can be one of:
1006                     'b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64'
1007                 'content' (mandatory): string with the content of the file
1008                 'permissions': (optional) string with file permissions, typically octal notation '0644'
1009                 'owner': (optional) file owner, string with the format 'owner:group'
1010             'boot-data-drive': boolean to indicate if user-data must be passed using a boot drive (hard disk)
1011             'disk_list': (optional) list with additional disks to the VM. Each item is a dict with:
1012                 'image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted
1013                 'size': (mandatory) string with the size of the disk in GB
1014                 'vim_id' (optional) should use this existing volume id
1015             availability_zone_index: Index of availability_zone_list to use for this this VM. None if not AV required
1016             availability_zone_list: list of availability zones given by user in the VNFD descriptor.  Ignore if
1017                 availability_zone_index is None
1018                 #TODO ip, security groups
1019         Returns a tuple with the instance identifier and created_items or raises an exception on error
1020             created_items can be None or a dictionary where this method can include key-values that will be passed to
1021             the method delete_vminstance and action_vminstance. Can be used to store created ports, volumes, etc.
1022             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
1023             as not present.
1024         """
1025         self.logger.debug("new_vminstance input: image='%s' flavor='%s' nics='%s'",image_id, flavor_id,str(net_list))
1026         try:
1027             server = None
1028             created_items = {}
1029             # metadata = {}
1030             net_list_vim = []
1031             external_network = []   # list of external networks to be connected to instance, later on used to create floating_ip
1032             no_secured_ports = []   # List of port-is with port-security disabled
1033             self._reload_connection()
1034             # metadata_vpci = {}   # For a specific neutron plugin
1035             block_device_mapping = None
1036             for net in net_list:
1037                 if not net.get("net_id"):   # skip non connected iface
1038                     continue
1039
1040                 port_dict={
1041                     "network_id": net["net_id"],
1042                     "name": net.get("name"),
1043                     "admin_state_up": True
1044                 }
1045                 if net["type"]=="virtual":
1046                     pass
1047                     # if "vpci" in net:
1048                     #     metadata_vpci[ net["net_id"] ] = [[ net["vpci"], "" ]]
1049                 elif net["type"] == "VF" or net["type"] == "SR-IOV":  # for VF
1050                     # if "vpci" in net:
1051                     #     if "VF" not in metadata_vpci:
1052                     #         metadata_vpci["VF"]=[]
1053                     #     metadata_vpci["VF"].append([ net["vpci"], "" ])
1054                     port_dict["binding:vnic_type"]="direct"
1055                     # VIO specific Changes
1056                     if self.vim_type == "VIO":
1057                         # Need to create port with port_security_enabled = False and no-security-groups
1058                         port_dict["port_security_enabled"]=False
1059                         port_dict["provider_security_groups"]=[]
1060                         port_dict["security_groups"]=[]
1061                 else:   # For PT PCI-PASSTHROUGH
1062                     # VIO specific Changes
1063                     # Current VIO release does not support port with type 'direct-physical'
1064                     # So no need to create virtual port in case of PCI-device.
1065                     # Will update port_dict code when support gets added in next VIO release
1066                     if self.vim_type == "VIO":
1067                         raise vimconn.vimconnNotSupportedException(
1068                             "Current VIO release does not support full passthrough (PT)")
1069                     # if "vpci" in net:
1070                     #     if "PF" not in metadata_vpci:
1071                     #         metadata_vpci["PF"]=[]
1072                     #     metadata_vpci["PF"].append([ net["vpci"], "" ])
1073                     port_dict["binding:vnic_type"]="direct-physical"
1074                 if not port_dict["name"]:
1075                     port_dict["name"]=name
1076                 if net.get("mac_address"):
1077                     port_dict["mac_address"]=net["mac_address"]
1078                 if net.get("ip_address"):
1079                     port_dict["fixed_ips"] = [{'ip_address': net["ip_address"]}]
1080                     # TODO add 'subnet_id': <subnet_id>
1081                 new_port = self.neutron.create_port({"port": port_dict })
1082                 created_items["port:" + str(new_port["port"]["id"])] = True
1083                 net["mac_adress"] = new_port["port"]["mac_address"]
1084                 net["vim_id"] = new_port["port"]["id"]
1085                 # if try to use a network without subnetwork, it will return a emtpy list
1086                 fixed_ips = new_port["port"].get("fixed_ips")
1087                 if fixed_ips:
1088                     net["ip"] = fixed_ips[0].get("ip_address")
1089                 else:
1090                     net["ip"] = None
1091
1092                 port = {"port-id": new_port["port"]["id"]}
1093                 if float(self.nova.api_version.get_string()) >= 2.32:
1094                     port["tag"] = new_port["port"]["name"]
1095                 net_list_vim.append(port)
1096
1097                 if net.get('floating_ip', False):
1098                     net['exit_on_floating_ip_error'] = True
1099                     external_network.append(net)
1100                 elif net['use'] == 'mgmt' and self.config.get('use_floating_ip'):
1101                     net['exit_on_floating_ip_error'] = False
1102                     external_network.append(net)
1103                     net['floating_ip'] = self.config.get('use_floating_ip')
1104
1105                 # If port security is disabled when the port has not yet been attached to the VM, then all vm traffic is dropped.
1106                 # As a workaround we wait until the VM is active and then disable the port-security
1107                 if net.get("port_security") == False and not self.config.get("no_port_security_extension"):
1108                     no_secured_ports.append(new_port["port"]["id"])
1109
1110             # if metadata_vpci:
1111             #     metadata = {"pci_assignement": json.dumps(metadata_vpci)}
1112             #     if len(metadata["pci_assignement"]) >255:
1113             #         #limit the metadata size
1114             #         #metadata["pci_assignement"] = metadata["pci_assignement"][0:255]
1115             #         self.logger.warn("Metadata deleted since it exceeds the expected length (255) ")
1116             #         metadata = {}
1117
1118             self.logger.debug("name '%s' image_id '%s'flavor_id '%s' net_list_vim '%s' description '%s'",
1119                               name, image_id, flavor_id, str(net_list_vim), description)
1120
1121             security_groups = self.config.get('security_groups')
1122             if type(security_groups) is str:
1123                 security_groups = ( security_groups, )
1124             # cloud config
1125             config_drive, userdata = self._create_user_data(cloud_config)
1126
1127             # Create additional volumes in case these are present in disk_list
1128             base_disk_index = ord('b')
1129             if disk_list:
1130                 block_device_mapping = {}
1131                 for disk in disk_list:
1132                     if disk.get('vim_id'):
1133                         block_device_mapping['_vd' + chr(base_disk_index)] = disk['vim_id']
1134                     else:
1135                         if 'image_id' in disk:
1136                             volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1137                                                                 chr(base_disk_index), imageRef=disk['image_id'])
1138                         else:
1139                             volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1140                                                                 chr(base_disk_index))
1141                         created_items["volume:" + str(volume.id)] = True
1142                         block_device_mapping['_vd' + chr(base_disk_index)] = volume.id
1143                     base_disk_index += 1
1144
1145                 # Wait until created volumes are with status available
1146                 elapsed_time = 0
1147                 while elapsed_time < volume_timeout:
1148                     for created_item in created_items:
1149                         v, _, volume_id = created_item.partition(":")
1150                         if v == 'volume':
1151                             if self.cinder.volumes.get(volume_id).status != 'available':
1152                                 break
1153                     else:  # all ready: break from while
1154                         break
1155                     time.sleep(5)
1156                     elapsed_time += 5
1157                 # If we exceeded the timeout rollback
1158                 if elapsed_time >= volume_timeout:
1159                     raise vimconn.vimconnException('Timeout creating volumes for instance ' + name,
1160                                                    http_code=vimconn.HTTP_Request_Timeout)
1161             # get availability Zone
1162             vm_av_zone = self._get_vm_availability_zone(availability_zone_index, availability_zone_list)
1163
1164             self.logger.debug("nova.servers.create({}, {}, {}, nics={}, security_groups={}, "
1165                               "availability_zone={}, key_name={}, userdata={}, config_drive={}, "
1166                               "block_device_mapping={})".format(name, image_id, flavor_id, net_list_vim,
1167                                                                 security_groups, vm_av_zone, self.config.get('keypair'),
1168                                                                 userdata, config_drive, block_device_mapping))
1169             server = self.nova.servers.create(name, image_id, flavor_id, nics=net_list_vim,
1170                                               security_groups=security_groups,
1171                                               availability_zone=vm_av_zone,
1172                                               key_name=self.config.get('keypair'),
1173                                               userdata=userdata,
1174                                               config_drive=config_drive,
1175                                               block_device_mapping=block_device_mapping
1176                                               )  # , description=description)
1177
1178             vm_start_time = time.time()
1179             # Previously mentioned workaround to wait until the VM is active and then disable the port-security
1180             if no_secured_ports:
1181                 self.__wait_for_vm(server.id, 'ACTIVE')
1182
1183             for port_id in no_secured_ports:
1184                 try:
1185                     self.neutron.update_port(port_id,
1186                                              {"port": {"port_security_enabled": False, "security_groups": None}})
1187                 except Exception as e:
1188                     raise vimconn.vimconnException("It was not possible to disable port security for port {}".format(
1189                         port_id))
1190             # print "DONE :-)", server
1191
1192             # pool_id = None
1193             if external_network:
1194                 floating_ips = self.neutron.list_floatingips().get("floatingips", ())
1195             for floating_network in external_network:
1196                 try:
1197                     assigned = False
1198                     while not assigned:
1199                         if floating_ips:
1200                             ip = floating_ips.pop(0)
1201                             if ip.get("port_id", False) or ip.get('tenant_id') != server.tenant_id:
1202                                 continue
1203                             if isinstance(floating_network['floating_ip'], str):
1204                                 if ip.get("floating_network_id") != floating_network['floating_ip']:
1205                                     continue
1206                             free_floating_ip = ip.get("floating_ip_address")
1207                         else:
1208                             if isinstance(floating_network['floating_ip'], str) and \
1209                                 floating_network['floating_ip'].lower() != "true":
1210                                 pool_id = floating_network['floating_ip']
1211                             else:
1212                                 # Find the external network
1213                                 external_nets = list()
1214                                 for net in self.neutron.list_networks()['networks']:
1215                                     if net['router:external']:
1216                                             external_nets.append(net)
1217
1218                                 if len(external_nets) == 0:
1219                                     raise vimconn.vimconnException("Cannot create floating_ip automatically since no external "
1220                                                                    "network is present",
1221                                                                     http_code=vimconn.HTTP_Conflict)
1222                                 if len(external_nets) > 1:
1223                                     raise vimconn.vimconnException("Cannot create floating_ip automatically since multiple "
1224                                                                    "external networks are present",
1225                                                                    http_code=vimconn.HTTP_Conflict)
1226
1227                                 pool_id = external_nets[0].get('id')
1228                             param = {'floatingip': {'floating_network_id': pool_id, 'tenant_id': server.tenant_id}}
1229                             try:
1230                                 # self.logger.debug("Creating floating IP")
1231                                 new_floating_ip = self.neutron.create_floatingip(param)
1232                                 free_floating_ip = new_floating_ip['floatingip']['floating_ip_address']
1233                             except Exception as e:
1234                                 raise vimconn.vimconnException(type(e).__name__ + ": Cannot create new floating_ip " +
1235                                                                str(e), http_code=vimconn.HTTP_Conflict)
1236
1237                         fix_ip = floating_network.get('ip')
1238                         while not assigned:
1239                             try:
1240                                 server.add_floating_ip(free_floating_ip, fix_ip)
1241                                 assigned = True
1242                             except Exception as e:
1243                                 # openstack need some time after VM creation to asign an IP. So retry if fails
1244                                 vm_status = self.nova.servers.get(server.id).status
1245                                 if vm_status != 'ACTIVE' and vm_status != 'ERROR':
1246                                     if time.time() - vm_start_time < server_timeout:
1247                                         time.sleep(5)
1248                                         continue
1249                                 raise vimconn.vimconnException(
1250                                     "Cannot create floating_ip: {} {}".format(type(e).__name__, e),
1251                                     http_code=vimconn.HTTP_Conflict)
1252
1253                 except Exception as e:
1254                     if not floating_network['exit_on_floating_ip_error']:
1255                         self.logger.warn("Cannot create floating_ip. %s", str(e))
1256                         continue
1257                     raise
1258
1259             return server.id, created_items
1260 #        except nvExceptions.NotFound as e:
1261 #            error_value=-vimconn.HTTP_Not_Found
1262 #            error_text= "vm instance %s not found" % vm_id
1263 #        except TypeError as e:
1264 #            raise vimconn.vimconnException(type(e).__name__ + ": "+  str(e), http_code=vimconn.HTTP_Bad_Request)
1265
1266         except Exception as e:
1267             server_id = None
1268             if server:
1269                 server_id = server.id
1270             try:
1271                 self.delete_vminstance(server_id, created_items)
1272             except Exception as e2:
1273                 self.logger.error("new_vminstance rollback fail {}".format(e2))
1274
1275             self._format_exception(e)
1276
1277     def get_vminstance(self,vm_id):
1278         '''Returns the VM instance information from VIM'''
1279         #self.logger.debug("Getting VM from VIM")
1280         try:
1281             self._reload_connection()
1282             server = self.nova.servers.find(id=vm_id)
1283             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
1284             return server.to_dict()
1285         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1286             self._format_exception(e)
1287
1288     def get_vminstance_console(self,vm_id, console_type="vnc"):
1289         '''
1290         Get a console for the virtual machine
1291         Params:
1292             vm_id: uuid of the VM
1293             console_type, can be:
1294                 "novnc" (by default), "xvpvnc" for VNC types,
1295                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
1296         Returns dict with the console parameters:
1297                 protocol: ssh, ftp, http, https, ...
1298                 server:   usually ip address
1299                 port:     the http, ssh, ... port
1300                 suffix:   extra text, e.g. the http path and query string
1301         '''
1302         self.logger.debug("Getting VM CONSOLE from VIM")
1303         try:
1304             self._reload_connection()
1305             server = self.nova.servers.find(id=vm_id)
1306             if console_type == None or console_type == "novnc":
1307                 console_dict = server.get_vnc_console("novnc")
1308             elif console_type == "xvpvnc":
1309                 console_dict = server.get_vnc_console(console_type)
1310             elif console_type == "rdp-html5":
1311                 console_dict = server.get_rdp_console(console_type)
1312             elif console_type == "spice-html5":
1313                 console_dict = server.get_spice_console(console_type)
1314             else:
1315                 raise vimconn.vimconnException("console type '{}' not allowed".format(console_type), http_code=vimconn.HTTP_Bad_Request)
1316
1317             console_dict1 = console_dict.get("console")
1318             if console_dict1:
1319                 console_url = console_dict1.get("url")
1320                 if console_url:
1321                     #parse console_url
1322                     protocol_index = console_url.find("//")
1323                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1324                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1325                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1326                         return -vimconn.HTTP_Internal_Server_Error, "Unexpected response from VIM"
1327                     console_dict={"protocol": console_url[0:protocol_index],
1328                                   "server":   console_url[protocol_index+2:port_index],
1329                                   "port":     console_url[port_index:suffix_index],
1330                                   "suffix":   console_url[suffix_index+1:]
1331                                   }
1332                     protocol_index += 2
1333                     return console_dict
1334             raise vimconn.vimconnUnexpectedResponse("Unexpected response from VIM")
1335
1336         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.BadRequest, ConnectionError) as e:
1337             self._format_exception(e)
1338
1339     def delete_vminstance(self, vm_id, created_items=None):
1340         '''Removes a VM instance from VIM. Returns the old identifier
1341         '''
1342         #print "osconnector: Getting VM from VIM"
1343         if created_items == None:
1344             created_items = {}
1345         try:
1346             self._reload_connection()
1347             # delete VM ports attached to this networks before the virtual machine
1348             for k, v in created_items.items():
1349                 if not v:  # skip already deleted
1350                     continue
1351                 try:
1352                     k_item, _, k_id = k.partition(":")
1353                     if k_item == "port":
1354                         self.neutron.delete_port(k_id)
1355                 except Exception as e:
1356                     self.logger.error("Error deleting port: {}: {}".format(type(e).__name__, e))
1357
1358             # #commented because detaching the volumes makes the servers.delete not work properly ?!?
1359             # #dettach volumes attached
1360             # server = self.nova.servers.get(vm_id)
1361             # volumes_attached_dict = server._info['os-extended-volumes:volumes_attached']   #volume['id']
1362             # #for volume in volumes_attached_dict:
1363             # #    self.cinder.volumes.detach(volume['id'])
1364
1365             if vm_id:
1366                 self.nova.servers.delete(vm_id)
1367
1368             # delete volumes. Although having detached, they should have in active status before deleting
1369             # we ensure in this loop
1370             keep_waiting = True
1371             elapsed_time = 0
1372             while keep_waiting and elapsed_time < volume_timeout:
1373                 keep_waiting = False
1374                 for k, v in created_items.items():
1375                     if not v:  # skip already deleted
1376                         continue
1377                     try:
1378                         k_item, _, k_id = k.partition(":")
1379                         if k_item == "volume":
1380                             if self.cinder.volumes.get(k_id).status != 'available':
1381                                 keep_waiting = True
1382                             else:
1383                                 self.cinder.volumes.delete(k_id)
1384                     except Exception as e:
1385                         self.logger.error("Error deleting volume: {}: {}".format(type(e).__name__, e))
1386                 if keep_waiting:
1387                     time.sleep(1)
1388                     elapsed_time += 1
1389             return None
1390         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
1391             self._format_exception(e)
1392
1393     def refresh_vms_status(self, vm_list):
1394         '''Get the status of the virtual machines and their interfaces/ports
1395            Params: the list of VM identifiers
1396            Returns a dictionary with:
1397                 vm_id:          #VIM id of this Virtual Machine
1398                     status:     #Mandatory. Text with one of:
1399                                 #  DELETED (not found at vim)
1400                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1401                                 #  OTHER (Vim reported other status not understood)
1402                                 #  ERROR (VIM indicates an ERROR status)
1403                                 #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
1404                                 #  CREATING (on building process), ERROR
1405                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
1406                                 #
1407                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
1408                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
1409                     interfaces:
1410                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
1411                         mac_address:      #Text format XX:XX:XX:XX:XX:XX
1412                         vim_net_id:       #network id where this interface is connected
1413                         vim_interface_id: #interface/port VIM id
1414                         ip_address:       #null, or text with IPv4, IPv6 address
1415                         compute_node:     #identification of compute node where PF,VF interface is allocated
1416                         pci:              #PCI address of the NIC that hosts the PF,VF
1417                         vlan:             #physical VLAN used for VF
1418         '''
1419         vm_dict={}
1420         self.logger.debug("refresh_vms status: Getting tenant VM instance information from VIM")
1421         for vm_id in vm_list:
1422             vm={}
1423             try:
1424                 vm_vim = self.get_vminstance(vm_id)
1425                 if vm_vim['status'] in vmStatus2manoFormat:
1426                     vm['status']    =  vmStatus2manoFormat[ vm_vim['status'] ]
1427                 else:
1428                     vm['status']    = "OTHER"
1429                     vm['error_msg'] = "VIM status reported " + vm_vim['status']
1430
1431                 vm['vim_info'] = self.serialize(vm_vim)
1432
1433                 vm["interfaces"] = []
1434                 if vm_vim.get('fault'):
1435                     vm['error_msg'] = str(vm_vim['fault'])
1436                 #get interfaces
1437                 try:
1438                     self._reload_connection()
1439                     port_dict = self.neutron.list_ports(device_id=vm_id)
1440                     for port in port_dict["ports"]:
1441                         interface={}
1442                         interface['vim_info'] = self.serialize(port)
1443                         interface["mac_address"] = port.get("mac_address")
1444                         interface["vim_net_id"] = port["network_id"]
1445                         interface["vim_interface_id"] = port["id"]
1446                         # check if OS-EXT-SRV-ATTR:host is there,
1447                         # in case of non-admin credentials, it will be missing
1448                         if vm_vim.get('OS-EXT-SRV-ATTR:host'):
1449                             interface["compute_node"] = vm_vim['OS-EXT-SRV-ATTR:host']
1450                         interface["pci"] = None
1451
1452                         # check if binding:profile is there,
1453                         # in case of non-admin credentials, it will be missing
1454                         if port.get('binding:profile'):
1455                             if port['binding:profile'].get('pci_slot'):
1456                                 # TODO: At the moment sr-iov pci addresses are converted to PF pci addresses by setting the slot to 0x00
1457                                 # TODO: This is just a workaround valid for niantinc. Find a better way to do so
1458                                 #   CHANGE DDDD:BB:SS.F to DDDD:BB:00.(F%2)   assuming there are 2 ports per nic
1459                                 pci = port['binding:profile']['pci_slot']
1460                                 # interface["pci"] = pci[:-4] + "00." + str(int(pci[-1]) % 2)
1461                                 interface["pci"] = pci
1462                         interface["vlan"] = None
1463                         #if network is of type vlan and port is of type direct (sr-iov) then set vlan id
1464                         network = self.neutron.show_network(port["network_id"])
1465                         if network['network'].get('provider:network_type') == 'vlan' and \
1466                             port.get("binding:vnic_type") == "direct":
1467                             interface["vlan"] = network['network'].get('provider:segmentation_id')
1468                         ips=[]
1469                         #look for floating ip address
1470                         try:
1471                             floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
1472                             if floating_ip_dict.get("floatingips"):
1473                                 ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address") )
1474                         except Exception:
1475                             pass
1476
1477                         for subnet in port["fixed_ips"]:
1478                             ips.append(subnet["ip_address"])
1479                         interface["ip_address"] = ";".join(ips)
1480                         vm["interfaces"].append(interface)
1481                 except Exception as e:
1482                     self.logger.error("Error getting vm interface information {}: {}".format(type(e).__name__, e),
1483                                       exc_info=True)
1484             except vimconn.vimconnNotFoundException as e:
1485                 self.logger.error("Exception getting vm status: %s", str(e))
1486                 vm['status'] = "DELETED"
1487                 vm['error_msg'] = str(e)
1488             except vimconn.vimconnException as e:
1489                 self.logger.error("Exception getting vm status: %s", str(e))
1490                 vm['status'] = "VIM_ERROR"
1491                 vm['error_msg'] = str(e)
1492             vm_dict[vm_id] = vm
1493         return vm_dict
1494
1495     def action_vminstance(self, vm_id, action_dict, created_items={}):
1496         '''Send and action over a VM instance from VIM
1497         Returns None or the console dict if the action was successfully sent to the VIM'''
1498         self.logger.debug("Action over VM '%s': %s", vm_id, str(action_dict))
1499         try:
1500             self._reload_connection()
1501             server = self.nova.servers.find(id=vm_id)
1502             if "start" in action_dict:
1503                 if action_dict["start"]=="rebuild":
1504                     server.rebuild()
1505                 else:
1506                     if server.status=="PAUSED":
1507                         server.unpause()
1508                     elif server.status=="SUSPENDED":
1509                         server.resume()
1510                     elif server.status=="SHUTOFF":
1511                         server.start()
1512             elif "pause" in action_dict:
1513                 server.pause()
1514             elif "resume" in action_dict:
1515                 server.resume()
1516             elif "shutoff" in action_dict or "shutdown" in action_dict:
1517                 server.stop()
1518             elif "forceOff" in action_dict:
1519                 server.stop() #TODO
1520             elif "terminate" in action_dict:
1521                 server.delete()
1522             elif "createImage" in action_dict:
1523                 server.create_image()
1524                 #"path":path_schema,
1525                 #"description":description_schema,
1526                 #"name":name_schema,
1527                 #"metadata":metadata_schema,
1528                 #"imageRef": id_schema,
1529                 #"disk": {"oneOf":[{"type": "null"}, {"type":"string"}] },
1530             elif "rebuild" in action_dict:
1531                 server.rebuild(server.image['id'])
1532             elif "reboot" in action_dict:
1533                 server.reboot() #reboot_type='SOFT'
1534             elif "console" in action_dict:
1535                 console_type = action_dict["console"]
1536                 if console_type == None or console_type == "novnc":
1537                     console_dict = server.get_vnc_console("novnc")
1538                 elif console_type == "xvpvnc":
1539                     console_dict = server.get_vnc_console(console_type)
1540                 elif console_type == "rdp-html5":
1541                     console_dict = server.get_rdp_console(console_type)
1542                 elif console_type == "spice-html5":
1543                     console_dict = server.get_spice_console(console_type)
1544                 else:
1545                     raise vimconn.vimconnException("console type '{}' not allowed".format(console_type),
1546                                                    http_code=vimconn.HTTP_Bad_Request)
1547                 try:
1548                     console_url = console_dict["console"]["url"]
1549                     #parse console_url
1550                     protocol_index = console_url.find("//")
1551                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1552                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1553                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1554                         raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1555                     console_dict2={"protocol": console_url[0:protocol_index],
1556                                   "server":   console_url[protocol_index+2 : port_index],
1557                                   "port":     int(console_url[port_index+1 : suffix_index]),
1558                                   "suffix":   console_url[suffix_index+1:]
1559                                   }
1560                     return console_dict2
1561                 except Exception as e:
1562                     raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1563
1564             return None
1565         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1566             self._format_exception(e)
1567         #TODO insert exception vimconn.HTTP_Unauthorized
1568
1569     ####### VIO Specific Changes #########
1570     def _genrate_vlanID(self):
1571         """
1572          Method to get unused vlanID
1573             Args:
1574                 None
1575             Returns:
1576                 vlanID
1577         """
1578         #Get used VLAN IDs
1579         usedVlanIDs = []
1580         networks = self.get_network_list()
1581         for net in networks:
1582             if net.get('provider:segmentation_id'):
1583                 usedVlanIDs.append(net.get('provider:segmentation_id'))
1584         used_vlanIDs = set(usedVlanIDs)
1585
1586         #find unused VLAN ID
1587         for vlanID_range in self.config.get('dataplane_net_vlan_range'):
1588             try:
1589                 start_vlanid , end_vlanid = map(int, vlanID_range.replace(" ", "").split("-"))
1590                 for vlanID in xrange(start_vlanid, end_vlanid + 1):
1591                     if vlanID not in used_vlanIDs:
1592                         return vlanID
1593             except Exception as exp:
1594                 raise vimconn.vimconnException("Exception {} occurred while generating VLAN ID.".format(exp))
1595         else:
1596             raise vimconn.vimconnConflictException("Unable to create the SRIOV VLAN network."\
1597                 " All given Vlan IDs {} are in use.".format(self.config.get('dataplane_net_vlan_range')))
1598
1599
1600     def _validate_vlan_ranges(self, dataplane_net_vlan_range):
1601         """
1602         Method to validate user given vlanID ranges
1603             Args:  None
1604             Returns: None
1605         """
1606         for vlanID_range in dataplane_net_vlan_range:
1607             vlan_range = vlanID_range.replace(" ", "")
1608             #validate format
1609             vlanID_pattern = r'(\d)*-(\d)*$'
1610             match_obj = re.match(vlanID_pattern, vlan_range)
1611             if not match_obj:
1612                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}.You must provide "\
1613                 "'dataplane_net_vlan_range' in format [start_ID - end_ID].".format(vlanID_range))
1614
1615             start_vlanid , end_vlanid = map(int,vlan_range.split("-"))
1616             if start_vlanid <= 0 :
1617                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1618                 "Start ID can not be zero. For VLAN "\
1619                 "networks valid IDs are 1 to 4094 ".format(vlanID_range))
1620             if end_vlanid > 4094 :
1621                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1622                 "End VLAN ID can not be greater than 4094. For VLAN "\
1623                 "networks valid IDs are 1 to 4094 ".format(vlanID_range))
1624
1625             if start_vlanid > end_vlanid:
1626                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1627                     "You must provide a 'dataplane_net_vlan_range' in format start_ID - end_ID and "\
1628                     "start_ID < end_ID ".format(vlanID_range))
1629
1630 #NOT USED FUNCTIONS
1631
1632     def new_external_port(self, port_data):
1633         #TODO openstack if needed
1634         '''Adds a external port to VIM'''
1635         '''Returns the port identifier'''
1636         return -vimconn.HTTP_Internal_Server_Error, "osconnector.new_external_port() not implemented"
1637
1638     def connect_port_network(self, port_id, network_id, admin=False):
1639         #TODO openstack if needed
1640         '''Connects a external port to a network'''
1641         '''Returns status code of the VIM response'''
1642         return -vimconn.HTTP_Internal_Server_Error, "osconnector.connect_port_network() not implemented"
1643
1644     def new_user(self, user_name, user_passwd, tenant_id=None):
1645         '''Adds a new user to openstack VIM'''
1646         '''Returns the user identifier'''
1647         self.logger.debug("osconnector: Adding a new user to VIM")
1648         try:
1649             self._reload_connection()
1650             user=self.keystone.users.create(user_name, password=user_passwd, default_project=tenant_id)
1651             #self.keystone.tenants.add_user(self.k_creds["username"], #role)
1652             return user.id
1653         except ksExceptions.ConnectionError as e:
1654             error_value=-vimconn.HTTP_Bad_Request
1655             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1656         except ksExceptions.ClientException as e: #TODO remove
1657             error_value=-vimconn.HTTP_Bad_Request
1658             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1659         #TODO insert exception vimconn.HTTP_Unauthorized
1660         #if reaching here is because an exception
1661         self.logger.debug("new_user " + error_text)
1662         return error_value, error_text
1663
1664     def delete_user(self, user_id):
1665         '''Delete a user from openstack VIM'''
1666         '''Returns the user identifier'''
1667         if self.debug:
1668             print("osconnector: Deleting  a  user from VIM")
1669         try:
1670             self._reload_connection()
1671             self.keystone.users.delete(user_id)
1672             return 1, user_id
1673         except ksExceptions.ConnectionError as e:
1674             error_value=-vimconn.HTTP_Bad_Request
1675             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1676         except ksExceptions.NotFound as e:
1677             error_value=-vimconn.HTTP_Not_Found
1678             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1679         except ksExceptions.ClientException as e: #TODO remove
1680             error_value=-vimconn.HTTP_Bad_Request
1681             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1682         #TODO insert exception vimconn.HTTP_Unauthorized
1683         #if reaching here is because an exception
1684             self.logger.debug("delete_tenant " + error_text)
1685         return error_value, error_text
1686
1687     def get_hosts_info(self):
1688         '''Get the information of deployed hosts
1689         Returns the hosts content'''
1690         if self.debug:
1691             print("osconnector: Getting Host info from VIM")
1692         try:
1693             h_list=[]
1694             self._reload_connection()
1695             hypervisors = self.nova.hypervisors.list()
1696             for hype in hypervisors:
1697                 h_list.append( hype.to_dict() )
1698             return 1, {"hosts":h_list}
1699         except nvExceptions.NotFound as e:
1700             error_value=-vimconn.HTTP_Not_Found
1701             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1702         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1703             error_value=-vimconn.HTTP_Bad_Request
1704             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1705         #TODO insert exception vimconn.HTTP_Unauthorized
1706         #if reaching here is because an exception
1707         self.logger.debug("get_hosts_info " + error_text)
1708         return error_value, error_text
1709
1710     def get_hosts(self, vim_tenant):
1711         '''Get the hosts and deployed instances
1712         Returns the hosts content'''
1713         r, hype_dict = self.get_hosts_info()
1714         if r<0:
1715             return r, hype_dict
1716         hypervisors = hype_dict["hosts"]
1717         try:
1718             servers = self.nova.servers.list()
1719             for hype in hypervisors:
1720                 for server in servers:
1721                     if server.to_dict()['OS-EXT-SRV-ATTR:hypervisor_hostname']==hype['hypervisor_hostname']:
1722                         if 'vm' in hype:
1723                             hype['vm'].append(server.id)
1724                         else:
1725                             hype['vm'] = [server.id]
1726             return 1, hype_dict
1727         except nvExceptions.NotFound as e:
1728             error_value=-vimconn.HTTP_Not_Found
1729             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1730         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1731             error_value=-vimconn.HTTP_Bad_Request
1732             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1733         #TODO insert exception vimconn.HTTP_Unauthorized
1734         #if reaching here is because an exception
1735         self.logger.debug("get_hosts " + error_text)
1736         return error_value, error_text
1737
1738     def new_classification(self, name, ctype, definition):
1739         self.logger.debug(
1740             'Adding a new (Traffic) Classification to VIM, named %s', name)
1741         try:
1742             new_class = None
1743             self._reload_connection()
1744             if ctype not in supportedClassificationTypes:
1745                 raise vimconn.vimconnNotSupportedException(
1746                         'OpenStack VIM connector doesn\'t support provided '
1747                         'Classification Type {}, supported ones are: '
1748                         '{}'.format(ctype, supportedClassificationTypes))
1749             if not self._validate_classification(ctype, definition):
1750                 raise vimconn.vimconnException(
1751                     'Incorrect Classification definition '
1752                     'for the type specified.')
1753             classification_dict = definition
1754             classification_dict['name'] = name
1755
1756             new_class = self.neutron.create_sfc_flow_classifier(
1757                 {'flow_classifier': classification_dict})
1758             return new_class['flow_classifier']['id']
1759         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1760                 neExceptions.NeutronException, ConnectionError) as e:
1761             self.logger.error(
1762                 'Creation of Classification failed.')
1763             self._format_exception(e)
1764
1765     def get_classification(self, class_id):
1766         self.logger.debug(" Getting Classification %s from VIM", class_id)
1767         filter_dict = {"id": class_id}
1768         class_list = self.get_classification_list(filter_dict)
1769         if len(class_list) == 0:
1770             raise vimconn.vimconnNotFoundException(
1771                 "Classification '{}' not found".format(class_id))
1772         elif len(class_list) > 1:
1773             raise vimconn.vimconnConflictException(
1774                 "Found more than one Classification with this criteria")
1775         classification = class_list[0]
1776         return classification
1777
1778     def get_classification_list(self, filter_dict={}):
1779         self.logger.debug("Getting Classifications from VIM filter: '%s'",
1780                           str(filter_dict))
1781         try:
1782             filter_dict_os = filter_dict.copy()
1783             self._reload_connection()
1784             if self.api_version3 and "tenant_id" in filter_dict_os:
1785                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1786             classification_dict = self.neutron.list_sfc_flow_classifiers(
1787                 **filter_dict_os)
1788             classification_list = classification_dict["flow_classifiers"]
1789             self.__classification_os2mano(classification_list)
1790             return classification_list
1791         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1792                 neExceptions.NeutronException, ConnectionError) as e:
1793             self._format_exception(e)
1794
1795     def delete_classification(self, class_id):
1796         self.logger.debug("Deleting Classification '%s' from VIM", class_id)
1797         try:
1798             self._reload_connection()
1799             self.neutron.delete_sfc_flow_classifier(class_id)
1800             return class_id
1801         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1802                 ksExceptions.ClientException, neExceptions.NeutronException,
1803                 ConnectionError) as e:
1804             self._format_exception(e)
1805
1806     def new_sfi(self, name, ingress_ports, egress_ports, sfc_encap=True):
1807         self.logger.debug(
1808             "Adding a new Service Function Instance to VIM, named '%s'", name)
1809         try:
1810             new_sfi = None
1811             self._reload_connection()
1812             correlation = None
1813             if sfc_encap:
1814                 correlation = 'nsh'
1815             if len(ingress_ports) != 1:
1816                 raise vimconn.vimconnNotSupportedException(
1817                     "OpenStack VIM connector can only have "
1818                     "1 ingress port per SFI")
1819             if len(egress_ports) != 1:
1820                 raise vimconn.vimconnNotSupportedException(
1821                     "OpenStack VIM connector can only have "
1822                     "1 egress port per SFI")
1823             sfi_dict = {'name': name,
1824                         'ingress': ingress_ports[0],
1825                         'egress': egress_ports[0],
1826                         'service_function_parameters': {
1827                             'correlation': correlation}}
1828             new_sfi = self.neutron.create_sfc_port_pair({'port_pair': sfi_dict})
1829             return new_sfi['port_pair']['id']
1830         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1831                 neExceptions.NeutronException, ConnectionError) as e:
1832             if new_sfi:
1833                 try:
1834                     self.neutron.delete_sfc_port_pair(
1835                         new_sfi['port_pair']['id'])
1836                 except Exception:
1837                     self.logger.error(
1838                         'Creation of Service Function Instance failed, with '
1839                         'subsequent deletion failure as well.')
1840             self._format_exception(e)
1841
1842     def get_sfi(self, sfi_id):
1843         self.logger.debug(
1844             'Getting Service Function Instance %s from VIM', sfi_id)
1845         filter_dict = {"id": sfi_id}
1846         sfi_list = self.get_sfi_list(filter_dict)
1847         if len(sfi_list) == 0:
1848             raise vimconn.vimconnNotFoundException(
1849                 "Service Function Instance '{}' not found".format(sfi_id))
1850         elif len(sfi_list) > 1:
1851             raise vimconn.vimconnConflictException(
1852                 'Found more than one Service Function Instance '
1853                 'with this criteria')
1854         sfi = sfi_list[0]
1855         return sfi
1856
1857     def get_sfi_list(self, filter_dict={}):
1858         self.logger.debug("Getting Service Function Instances from "
1859                           "VIM filter: '%s'", str(filter_dict))
1860         try:
1861             self._reload_connection()
1862             filter_dict_os = filter_dict.copy()
1863             if self.api_version3 and "tenant_id" in filter_dict_os:
1864                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1865             sfi_dict = self.neutron.list_sfc_port_pairs(**filter_dict_os)
1866             sfi_list = sfi_dict["port_pairs"]
1867             self.__sfi_os2mano(sfi_list)
1868             return sfi_list
1869         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1870                 neExceptions.NeutronException, ConnectionError) as e:
1871             self._format_exception(e)
1872
1873     def delete_sfi(self, sfi_id):
1874         self.logger.debug("Deleting Service Function Instance '%s' "
1875                           "from VIM", sfi_id)
1876         try:
1877             self._reload_connection()
1878             self.neutron.delete_sfc_port_pair(sfi_id)
1879             return sfi_id
1880         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1881                 ksExceptions.ClientException, neExceptions.NeutronException,
1882                 ConnectionError) as e:
1883             self._format_exception(e)
1884
1885     def new_sf(self, name, sfis, sfc_encap=True):
1886         self.logger.debug("Adding a new Service Function to VIM, "
1887                           "named '%s'", name)
1888         try:
1889             new_sf = None
1890             self._reload_connection()
1891             # correlation = None
1892             # if sfc_encap:
1893             #     correlation = 'nsh'
1894             for instance in sfis:
1895                 sfi = self.get_sfi(instance)
1896                 if sfi.get('sfc_encap') != sfc_encap:
1897                     raise vimconn.vimconnNotSupportedException(
1898                         "OpenStack VIM connector requires all SFIs of the "
1899                         "same SF to share the same SFC Encapsulation")
1900             sf_dict = {'name': name,
1901                        'port_pairs': sfis}
1902             new_sf = self.neutron.create_sfc_port_pair_group({
1903                 'port_pair_group': sf_dict})
1904             return new_sf['port_pair_group']['id']
1905         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1906                 neExceptions.NeutronException, ConnectionError) as e:
1907             if new_sf:
1908                 try:
1909                     self.neutron.delete_sfc_port_pair_group(
1910                         new_sf['port_pair_group']['id'])
1911                 except Exception:
1912                     self.logger.error(
1913                         'Creation of Service Function failed, with '
1914                         'subsequent deletion failure as well.')
1915             self._format_exception(e)
1916
1917     def get_sf(self, sf_id):
1918         self.logger.debug("Getting Service Function %s from VIM", sf_id)
1919         filter_dict = {"id": sf_id}
1920         sf_list = self.get_sf_list(filter_dict)
1921         if len(sf_list) == 0:
1922             raise vimconn.vimconnNotFoundException(
1923                 "Service Function '{}' not found".format(sf_id))
1924         elif len(sf_list) > 1:
1925             raise vimconn.vimconnConflictException(
1926                 "Found more than one Service Function with this criteria")
1927         sf = sf_list[0]
1928         return sf
1929
1930     def get_sf_list(self, filter_dict={}):
1931         self.logger.debug("Getting Service Function from VIM filter: '%s'",
1932                           str(filter_dict))
1933         try:
1934             self._reload_connection()
1935             filter_dict_os = filter_dict.copy()
1936             if self.api_version3 and "tenant_id" in filter_dict_os:
1937                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1938             sf_dict = self.neutron.list_sfc_port_pair_groups(**filter_dict_os)
1939             sf_list = sf_dict["port_pair_groups"]
1940             self.__sf_os2mano(sf_list)
1941             return sf_list
1942         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1943                 neExceptions.NeutronException, ConnectionError) as e:
1944             self._format_exception(e)
1945
1946     def delete_sf(self, sf_id):
1947         self.logger.debug("Deleting Service Function '%s' from VIM", sf_id)
1948         try:
1949             self._reload_connection()
1950             self.neutron.delete_sfc_port_pair_group(sf_id)
1951             return sf_id
1952         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1953                 ksExceptions.ClientException, neExceptions.NeutronException,
1954                 ConnectionError) as e:
1955             self._format_exception(e)
1956
1957     def new_sfp(self, name, classifications, sfs, sfc_encap=True, spi=None):
1958         self.logger.debug("Adding a new Service Function Path to VIM, "
1959                           "named '%s'", name)
1960         try:
1961             new_sfp = None
1962             self._reload_connection()
1963             # In networking-sfc the MPLS encapsulation is legacy
1964             # should be used when no full SFC Encapsulation is intended
1965             sfc_encap = 'mpls'
1966             if sfc_encap:
1967                 correlation = 'nsh'
1968             sfp_dict = {'name': name,
1969                         'flow_classifiers': classifications,
1970                         'port_pair_groups': sfs,
1971                         'chain_parameters': {'correlation': correlation}}
1972             if spi:
1973                 sfp_dict['chain_id'] = spi
1974             new_sfp = self.neutron.create_sfc_port_chain({'port_chain': sfp_dict})
1975             return new_sfp["port_chain"]["id"]
1976         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1977                 neExceptions.NeutronException, ConnectionError) as e:
1978             if new_sfp:
1979                 try:
1980                     self.neutron.delete_sfc_port_chain(new_sfp['port_chain']['id'])
1981                 except Exception:
1982                     self.logger.error(
1983                         'Creation of Service Function Path failed, with '
1984                         'subsequent deletion failure as well.')
1985             self._format_exception(e)
1986
1987     def get_sfp(self, sfp_id):
1988         self.logger.debug(" Getting Service Function Path %s from VIM", sfp_id)
1989         filter_dict = {"id": sfp_id}
1990         sfp_list = self.get_sfp_list(filter_dict)
1991         if len(sfp_list) == 0:
1992             raise vimconn.vimconnNotFoundException(
1993                 "Service Function Path '{}' not found".format(sfp_id))
1994         elif len(sfp_list) > 1:
1995             raise vimconn.vimconnConflictException(
1996                 "Found more than one Service Function Path with this criteria")
1997         sfp = sfp_list[0]
1998         return sfp
1999
2000     def get_sfp_list(self, filter_dict={}):
2001         self.logger.debug("Getting Service Function Paths from VIM filter: "
2002                           "'%s'", str(filter_dict))
2003         try:
2004             self._reload_connection()
2005             filter_dict_os = filter_dict.copy()
2006             if self.api_version3 and "tenant_id" in filter_dict_os:
2007                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
2008             sfp_dict = self.neutron.list_sfc_port_chains(**filter_dict_os)
2009             sfp_list = sfp_dict["port_chains"]
2010             self.__sfp_os2mano(sfp_list)
2011             return sfp_list
2012         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2013                 neExceptions.NeutronException, ConnectionError) as e:
2014             self._format_exception(e)
2015
2016     def delete_sfp(self, sfp_id):
2017         self.logger.debug(
2018             "Deleting Service Function Path '%s' from VIM", sfp_id)
2019         try:
2020             self._reload_connection()
2021             self.neutron.delete_sfc_port_chain(sfp_id)
2022             return sfp_id
2023         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
2024                 ksExceptions.ClientException, neExceptions.NeutronException,
2025                 ConnectionError) as e:
2026             self._format_exception(e)