osm_ro/vimconn_openstack.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 osconnector implements all the methods to interact with openstack using the python-neutronclient.
  26
  27 For the VNF forwarding graph, The OpenStack VIM connector calls the
  28 networking-sfc Neutron extension methods, whose resources are mapped
  29 to the VIM connector's SFC resources as follows:
  30 - Classification (OSM) -> Flow Classifier (Neutron)
  31 - Service Function Instance (OSM) -> Port Pair (Neutron)
  32 - Service Function (OSM) -> Port Pair Group (Neutron)
  33 - Service Function Path (OSM) -> Port Chain (Neutron)
  34 '''
  35 __author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes, xFlow Research, Igor D.C."
  36 __date__  = "$22-sep-2017 23:59:59$"
  37
  38 import vimconn
  39 # import json
  40 import logging
  41 import netaddr
  42 import time
  43 import yaml
  44 import random
  45 import re
  46 import copy
  47
  48 from novaclient import client as nClient, exceptions as nvExceptions
  49 from keystoneauth1.identity import v2, v3
  50 from keystoneauth1 import session
  51 import keystoneclient.exceptions as ksExceptions
  52 import keystoneclient.v3.client as ksClient_v3
  53 import keystoneclient.v2_0.client as ksClient_v2
  54 from glanceclient import client as glClient
  55 import glanceclient.client as gl1Client
  56 import glanceclient.exc as gl1Exceptions
  57 from  cinderclient import client as cClient
  58 from httplib import HTTPException
  59 from neutronclient.neutron import client as neClient
  60 from neutronclient.common import exceptions as neExceptions
  61 from requests.exceptions import ConnectionError
  62
  63
  64 """contain the openstack virtual machine status to openmano status"""
  65 vmStatus2manoFormat={'ACTIVE':'ACTIVE',
  66                      'PAUSED':'PAUSED',
  67                      'SUSPENDED': 'SUSPENDED',
  68                      'SHUTOFF':'INACTIVE',
  69                      'BUILD':'BUILD',
  70                      'ERROR':'ERROR','DELETED':'DELETED'
  71                      }
  72 netStatus2manoFormat={'ACTIVE':'ACTIVE','PAUSED':'PAUSED','INACTIVE':'INACTIVE','BUILD':'BUILD','ERROR':'ERROR','DELETED':'DELETED'
  73                      }
  74
  75 supportedClassificationTypes = ['legacy_flow_classifier']
  76
  77 #global var to have a timeout creating and deleting volumes
  78 volume_timeout = 600
  79 server_timeout = 600
  80
  81 class vimconnector(vimconn.vimconnector):
  82     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None,
  83                  log_level=None, config={}, persistent_info={}):
  84         '''using common constructor parameters. In this case
  85         'url' is the keystone authorization url,
  86         'url_admin' is not use
  87         '''
  88         api_version = config.get('APIversion')
  89         if api_version and api_version not in ('v3.3', 'v2.0', '2', '3'):
  90             raise vimconn.vimconnException("Invalid value '{}' for config:APIversion. "
  91                                            "Allowed values are 'v3.3', 'v2.0', '2' or '3'".format(api_version))
  92         vim_type = config.get('vim_type')
  93         if vim_type and vim_type not in ('vio', 'VIO'):
  94             raise vimconn.vimconnException("Invalid value '{}' for config:vim_type."
  95                             "Allowed values are 'vio' or 'VIO'".format(vim_type))
  96
  97         if config.get('dataplane_net_vlan_range') is not None:
  98             #validate vlan ranges provided by user
  99             self._validate_vlan_ranges(config.get('dataplane_net_vlan_range'))
 100
 101         vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level,
 102                                       config)
 103
 104         if self.config.get("insecure") and self.config.get("ca_cert"):
 105             raise vimconn.vimconnException("options insecure and ca_cert are mutually exclusive")
 106         self.verify = True
 107         if self.config.get("insecure"):
 108             self.verify = False
 109         if self.config.get("ca_cert"):
 110             self.verify = self.config.get("ca_cert")
 111
 112         if not url:
 113             raise TypeError('url param can not be NoneType')
 114         self.persistent_info = persistent_info
 115         self.availability_zone = persistent_info.get('availability_zone', None)
 116         self.session = persistent_info.get('session', {'reload_client': True})
 117         self.nova = self.session.get('nova')
 118         self.neutron = self.session.get('neutron')
 119         self.cinder = self.session.get('cinder')
 120         self.glance = self.session.get('glance')
 121         self.glancev1 = self.session.get('glancev1')
 122         self.keystone = self.session.get('keystone')
 123         self.api_version3 = self.session.get('api_version3')
 124         self.vim_type = self.config.get("vim_type")
 125         if self.vim_type:
 126             self.vim_type = self.vim_type.upper()
 127         if self.config.get("use_internal_endpoint"):
 128             self.endpoint_type = "internalURL"
 129         else:
 130             self.endpoint_type = None
 131
 132         self.logger = logging.getLogger('openmano.vim.openstack')
 133
 134         ####### VIO Specific Changes #########
 135         if self.vim_type == "VIO":
 136             self.logger = logging.getLogger('openmano.vim.vio')
 137
 138         if log_level:
 139             self.logger.setLevel( getattr(logging, log_level))
 140
 141     def __getitem__(self, index):
 142         """Get individuals parameters.
 143         Throw KeyError"""
 144         if index == 'project_domain_id':
 145             return self.config.get("project_domain_id")
 146         elif index == 'user_domain_id':
 147             return self.config.get("user_domain_id")
 148         else:
 149             return vimconn.vimconnector.__getitem__(self, index)
 150
 151     def __setitem__(self, index, value):
 152         """Set individuals parameters and it is marked as dirty so to force connection reload.
 153         Throw KeyError"""
 154         if index == 'project_domain_id':
 155             self.config["project_domain_id"] = value
 156         elif index == 'user_domain_id':
 157                 self.config["user_domain_id"] = value
 158         else:
 159             vimconn.vimconnector.__setitem__(self, index, value)
 160         self.session['reload_client'] = True
 161
 162     def _reload_connection(self):
 163         '''Called before any operation, it check if credentials has changed
 164         Throw keystoneclient.apiclient.exceptions.AuthorizationFailure
 165         '''
 166         #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-)
 167         if self.session['reload_client']:
 168             if self.config.get('APIversion'):
 169                 self.api_version3 = self.config['APIversion'] == 'v3.3' or self.config['APIversion'] == '3'
 170             else:  # get from ending auth_url that end with v3 or with v2.0
 171                 self.api_version3 =  self.url.endswith("/v3") or self.url.endswith("/v3/")
 172             self.session['api_version3'] = self.api_version3
 173             if self.api_version3:
 174                 if self.config.get('project_domain_id') or self.config.get('project_domain_name'):
 175                     project_domain_id_default = None
 176                 else:
 177                     project_domain_id_default = 'default'
 178                 if self.config.get('user_domain_id') or self.config.get('user_domain_name'):
 179                     user_domain_id_default = None
 180                 else:
 181                     user_domain_id_default = 'default'
 182                 auth = v3.Password(auth_url=self.url,
 183                                    username=self.user,
 184                                    password=self.passwd,
 185                                    project_name=self.tenant_name,
 186                                    project_id=self.tenant_id,
 187                                    project_domain_id=self.config.get('project_domain_id', project_domain_id_default),
 188                                    user_domain_id=self.config.get('user_domain_id', user_domain_id_default),
 189                                    project_domain_name=self.config.get('project_domain_name'),
 190                                    user_domain_name=self.config.get('user_domain_name'))
 191             else:
 192                 auth = v2.Password(auth_url=self.url,
 193                                    username=self.user,
 194                                    password=self.passwd,
 195                                    tenant_name=self.tenant_name,
 196                                    tenant_id=self.tenant_id)
 197             sess = session.Session(auth=auth, verify=self.verify)
 198             if self.api_version3:
 199                 self.keystone = ksClient_v3.Client(session=sess, endpoint_type=self.endpoint_type)
 200             else:
 201                 self.keystone = ksClient_v2.Client(session=sess, endpoint_type=self.endpoint_type)
 202             self.session['keystone'] = self.keystone
 203             # In order to enable microversion functionality an explicit microversion must be specified in 'config'.
 204             # This implementation approach is due to the warning message in
 205             # https://developer.openstack.org/api-guide/compute/microversions.html
 206             # where it is stated that microversion backwards compatibility is not guaranteed and clients should
 207             # always require an specific microversion.
 208             # To be able to use 'device role tagging' functionality define 'microversion: 2.32' in datacenter config
 209             version = self.config.get("microversion")
 210             if not version:
 211                 version = "2.1"
 212             self.nova = self.session['nova'] = nClient.Client(str(version), session=sess, endpoint_type=self.endpoint_type)
 213             self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess, endpoint_type=self.endpoint_type)
 214             self.cinder = self.session['cinder'] = cClient.Client(2, session=sess, endpoint_type=self.endpoint_type)
 215             if self.endpoint_type == "internalURL":
 216                 glance_service_id = self.keystone.services.list(name="glance")[0].id
 217                 glance_endpoint = self.keystone.endpoints.list(glance_service_id, interface="internal")[0].url
 218             else:
 219                 glance_endpoint = None
 220             self.glance = self.session['glance'] = glClient.Client(2, session=sess, endpoint=glance_endpoint)
 221             #using version 1 of glance client in new_image()
 222             self.glancev1 = self.session['glancev1'] = glClient.Client('1', session=sess,
 223                                                                        endpoint=glance_endpoint)
 224             self.session['reload_client'] = False
 225             self.persistent_info['session'] = self.session
 226             # add availablity zone info inside  self.persistent_info
 227             self._set_availablity_zones()
 228             self.persistent_info['availability_zone'] = self.availability_zone
 229
 230     def __net_os2mano(self, net_list_dict):
 231         '''Transform the net openstack format to mano format
 232         net_list_dict can be a list of dict or a single dict'''
 233         if type(net_list_dict) is dict:
 234             net_list_=(net_list_dict,)
 235         elif type(net_list_dict) is list:
 236             net_list_=net_list_dict
 237         else:
 238             raise TypeError("param net_list_dict must be a list or a dictionary")
 239         for net in net_list_:
 240             if net.get('provider:network_type') == "vlan":
 241                 net['type']='data'
 242             else:
 243                 net['type']='bridge'
 244
 245     def __classification_os2mano(self, class_list_dict):
 246         """Transform the openstack format (Flow Classifier) to mano format
 247         (Classification) class_list_dict can be a list of dict or a single dict
 248         """
 249         if isinstance(class_list_dict, dict):
 250             class_list_ = [class_list_dict]
 251         elif isinstance(class_list_dict, list):
 252             class_list_ = class_list_dict
 253         else:
 254             raise TypeError(
 255                 "param class_list_dict must be a list or a dictionary")
 256         for classification in class_list_:
 257             id = classification.pop('id')
 258             name = classification.pop('name')
 259             description = classification.pop('description')
 260             project_id = classification.pop('project_id')
 261             tenant_id = classification.pop('tenant_id')
 262             original_classification = copy.deepcopy(classification)
 263             classification.clear()
 264             classification['ctype'] = 'legacy_flow_classifier'
 265             classification['definition'] = original_classification
 266             classification['id'] = id
 267             classification['name'] = name
 268             classification['description'] = description
 269             classification['project_id'] = project_id
 270             classification['tenant_id'] = tenant_id
 271
 272     def __sfi_os2mano(self, sfi_list_dict):
 273         """Transform the openstack format (Port Pair) to mano format (SFI)
 274         sfi_list_dict can be a list of dict or a single dict
 275         """
 276         if isinstance(sfi_list_dict, dict):
 277             sfi_list_ = [sfi_list_dict]
 278         elif isinstance(sfi_list_dict, list):
 279             sfi_list_ = sfi_list_dict
 280         else:
 281             raise TypeError(
 282                 "param sfi_list_dict must be a list or a dictionary")
 283         for sfi in sfi_list_:
 284             sfi['ingress_ports'] = []
 285             sfi['egress_ports'] = []
 286             if sfi.get('ingress'):
 287                 sfi['ingress_ports'].append(sfi['ingress'])
 288             if sfi.get('egress'):
 289                 sfi['egress_ports'].append(sfi['egress'])
 290             del sfi['ingress']
 291             del sfi['egress']
 292             params = sfi.get('service_function_parameters')
 293             sfc_encap = False
 294             if params:
 295                 correlation = params.get('correlation')
 296                 if correlation:
 297                     sfc_encap = True
 298             sfi['sfc_encap'] = sfc_encap
 299             del sfi['service_function_parameters']
 300
 301     def __sf_os2mano(self, sf_list_dict):
 302         """Transform the openstack format (Port Pair Group) to mano format (SF)
 303         sf_list_dict can be a list of dict or a single dict
 304         """
 305         if isinstance(sf_list_dict, dict):
 306             sf_list_ = [sf_list_dict]
 307         elif isinstance(sf_list_dict, list):
 308             sf_list_ = sf_list_dict
 309         else:
 310             raise TypeError(
 311                 "param sf_list_dict must be a list or a dictionary")
 312         for sf in sf_list_:
 313             del sf['port_pair_group_parameters']
 314             sf['sfis'] = sf['port_pairs']
 315             del sf['port_pairs']
 316
 317     def __sfp_os2mano(self, sfp_list_dict):
 318         """Transform the openstack format (Port Chain) to mano format (SFP)
 319         sfp_list_dict can be a list of dict or a single dict
 320         """
 321         if isinstance(sfp_list_dict, dict):
 322             sfp_list_ = [sfp_list_dict]
 323         elif isinstance(sfp_list_dict, list):
 324             sfp_list_ = sfp_list_dict
 325         else:
 326             raise TypeError(
 327                 "param sfp_list_dict must be a list or a dictionary")
 328         for sfp in sfp_list_:
 329             params = sfp.pop('chain_parameters')
 330             sfc_encap = False
 331             if params:
 332                 correlation = params.get('correlation')
 333                 if correlation:
 334                     sfc_encap = True
 335             sfp['sfc_encap'] = sfc_encap
 336             sfp['spi'] = sfp.pop('chain_id')
 337             sfp['classifications'] = sfp.pop('flow_classifiers')
 338             sfp['service_functions'] = sfp.pop('port_pair_groups')
 339
 340     # placeholder for now; read TODO note below
 341     def _validate_classification(self, type, definition):
 342         # only legacy_flow_classifier Type is supported at this point
 343         return True
 344         # TODO(igordcard): this method should be an abstract method of an
 345         # abstract Classification class to be implemented by the specific
 346         # Types. Also, abstract vimconnector should call the validation
 347         # method before the implemented VIM connectors are called.
 348
 349     def _format_exception(self, exception):
 350         '''Transform a keystone, nova, neutron  exception into a vimconn exception'''
 351         if isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError,
 352                                   ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed
 353                                   )):
 354             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 355         elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException,
 356                                     neExceptions.NeutronException, nvExceptions.BadRequest)):
 357             raise vimconn.vimconnUnexpectedResponse(type(exception).__name__ + ": " + str(exception))
 358         elif isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound)):
 359             raise vimconn.vimconnNotFoundException(type(exception).__name__ + ": " + str(exception))
 360         elif isinstance(exception, nvExceptions.Conflict):
 361             raise vimconn.vimconnConflictException(type(exception).__name__ + ": " + str(exception))
 362         elif isinstance(exception, vimconn.vimconnException):
 363             raise exception
 364         else:  # ()
 365             self.logger.error("General Exception " + str(exception), exc_info=True)
 366             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 367
 368     def get_tenant_list(self, filter_dict={}):
 369         '''Obtain tenants of VIM
 370         filter_dict can contain the following keys:
 371             name: filter by tenant name
 372             id: filter by tenant uuid/id
 373             <other VIM specific>
 374         Returns the tenant list of dictionaries: [{'name':'<name>, 'id':'<id>, ...}, ...]
 375         '''
 376         self.logger.debug("Getting tenants from VIM filter: '%s'", str(filter_dict))
 377         try:
 378             self._reload_connection()
 379             if self.api_version3:
 380                 project_class_list = self.keystone.projects.list(name=filter_dict.get("name"))
 381             else:
 382                 project_class_list = self.keystone.tenants.findall(**filter_dict)
 383             project_list=[]
 384             for project in project_class_list:
 385                 if filter_dict.get('id') and filter_dict["id"] != project.id:
 386                     continue
 387                 project_list.append(project.to_dict())
 388             return project_list
 389         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError) as e:
 390             self._format_exception(e)
 391
 392     def new_tenant(self, tenant_name, tenant_description):
 393         '''Adds a new tenant to openstack VIM. Returns the tenant identifier'''
 394         self.logger.debug("Adding a new tenant name: %s", tenant_name)
 395         try:
 396             self._reload_connection()
 397             if self.api_version3:
 398                 project = self.keystone.projects.create(tenant_name, self.config.get("project_domain_id", "default"),
 399                                                         description=tenant_description, is_domain=False)
 400             else:
 401                 project = self.keystone.tenants.create(tenant_name, tenant_description)
 402             return project.id
 403         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 404             self._format_exception(e)
 405
 406     def delete_tenant(self, tenant_id):
 407         '''Delete a tenant from openstack VIM. Returns the old tenant identifier'''
 408         self.logger.debug("Deleting tenant %s from VIM", tenant_id)
 409         try:
 410             self._reload_connection()
 411             if self.api_version3:
 412                 self.keystone.projects.delete(tenant_id)
 413             else:
 414                 self.keystone.tenants.delete(tenant_id)
 415             return tenant_id
 416         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 417             self._format_exception(e)
 418
 419     def new_network(self,net_name, net_type, ip_profile=None, shared=False, vlan=None):
 420         '''Adds a tenant network to VIM. Returns the network identifier'''
 421         self.logger.debug("Adding a new network to VIM name '%s', type '%s'", net_name, net_type)
 422         #self.logger.debug(">>>>>>>>>>>>>>>>>> IP profile %s", str(ip_profile))
 423         try:
 424             new_net = None
 425             self._reload_connection()
 426             network_dict = {'name': net_name, 'admin_state_up': True}
 427             if net_type=="data" or net_type=="ptp":
 428                 if self.config.get('dataplane_physical_net') == None:
 429                     raise vimconn.vimconnConflictException("You must provide a 'dataplane_physical_net' at config value before creating sriov network")
 430                 network_dict["provider:physical_network"] = self.config['dataplane_physical_net'] #"physnet_sriov" #TODO physical
 431                 network_dict["provider:network_type"]     = "vlan"
 432                 if vlan!=None:
 433                     network_dict["provider:network_type"] = vlan
 434
 435                 ####### VIO Specific Changes #########
 436                 if self.vim_type == "VIO":
 437                     if vlan is not None:
 438                         network_dict["provider:segmentation_id"] = vlan
 439                     else:
 440                         if self.config.get('dataplane_net_vlan_range') is None:
 441                             raise vimconn.vimconnConflictException("You must provide "\
 442                                 "'dataplane_net_vlan_range' in format [start_ID - end_ID]"\
 443                                 "at config value before creating sriov network with vlan tag")
 444
 445                         network_dict["provider:segmentation_id"] = self._genrate_vlanID()
 446
 447             network_dict["shared"]=shared
 448             new_net=self.neutron.create_network({'network':network_dict})
 449             #print new_net
 450             #create subnetwork, even if there is no profile
 451             if not ip_profile:
 452                 ip_profile = {}
 453             if not ip_profile.get('subnet_address'):
 454                 #Fake subnet is required
 455                 subnet_rand = random.randint(0, 255)
 456                 ip_profile['subnet_address'] = "192.168.{}.0/24".format(subnet_rand)
 457             if 'ip_version' not in ip_profile:
 458                 ip_profile['ip_version'] = "IPv4"
 459             subnet = {"name":net_name+"-subnet",
 460                     "network_id": new_net["network"]["id"],
 461                     "ip_version": 4 if ip_profile['ip_version']=="IPv4" else 6,
 462                     "cidr": ip_profile['subnet_address']
 463                     }
 464             # Gateway should be set to None if not needed. Otherwise openstack assigns one by default
 465             if ip_profile.get('gateway_address'):
 466                 subnet['gateway_ip'] = ip_profile['gateway_address']
 467             else:
 468                 subnet['gateway_ip'] = None
 469             if ip_profile.get('dns_address'):
 470                 subnet['dns_nameservers'] = ip_profile['dns_address'].split(";")
 471             if 'dhcp_enabled' in ip_profile:
 472                 subnet['enable_dhcp'] = False if \
 473                     ip_profile['dhcp_enabled']=="false" or ip_profile['dhcp_enabled']==False else True
 474             if ip_profile.get('dhcp_start_address'):
 475                 subnet['allocation_pools'] = []
 476                 subnet['allocation_pools'].append(dict())
 477                 subnet['allocation_pools'][0]['start'] = ip_profile['dhcp_start_address']
 478             if ip_profile.get('dhcp_count'):
 479                 #parts = ip_profile['dhcp_start_address'].split('.')
 480                 #ip_int = (int(parts[0]) << 24) + (int(parts[1]) << 16) + (int(parts[2]) << 8) + int(parts[3])
 481                 ip_int = int(netaddr.IPAddress(ip_profile['dhcp_start_address']))
 482                 ip_int += ip_profile['dhcp_count'] - 1
 483                 ip_str = str(netaddr.IPAddress(ip_int))
 484                 subnet['allocation_pools'][0]['end'] = ip_str
 485             #self.logger.debug(">>>>>>>>>>>>>>>>>> Subnet: %s", str(subnet))
 486             self.neutron.create_subnet({"subnet": subnet} )
 487             return new_net["network"]["id"]
 488         except Exception as e:
 489             if new_net:
 490                 self.neutron.delete_network(new_net['network']['id'])
 491             self._format_exception(e)
 492
 493     def get_network_list(self, filter_dict={}):
 494         '''Obtain tenant networks of VIM
 495         Filter_dict can be:
 496             name: network name
 497             id: network uuid
 498             shared: boolean
 499             tenant_id: tenant
 500             admin_state_up: boolean
 501             status: 'ACTIVE'
 502         Returns the network list of dictionaries
 503         '''
 504         self.logger.debug("Getting network from VIM filter: '%s'", str(filter_dict))
 505         try:
 506             self._reload_connection()
 507             filter_dict_os = filter_dict.copy()
 508             if self.api_version3 and "tenant_id" in filter_dict_os:
 509                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')  #T ODO check
 510             net_dict = self.neutron.list_networks(**filter_dict_os)
 511             net_list = net_dict["networks"]
 512             self.__net_os2mano(net_list)
 513             return net_list
 514         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 515             self._format_exception(e)
 516
 517     def get_network(self, net_id):
 518         '''Obtain details of network from VIM
 519         Returns the network information from a network id'''
 520         self.logger.debug(" Getting tenant network %s from VIM", net_id)
 521         filter_dict={"id": net_id}
 522         net_list = self.get_network_list(filter_dict)
 523         if len(net_list)==0:
 524             raise vimconn.vimconnNotFoundException("Network '{}' not found".format(net_id))
 525         elif len(net_list)>1:
 526             raise vimconn.vimconnConflictException("Found more than one network with this criteria")
 527         net = net_list[0]
 528         subnets=[]
 529         for subnet_id in net.get("subnets", () ):
 530             try:
 531                 subnet = self.neutron.show_subnet(subnet_id)
 532             except Exception as e:
 533                 self.logger.error("osconnector.get_network(): Error getting subnet %s %s" % (net_id, str(e)))
 534                 subnet = {"id": subnet_id, "fault": str(e)}
 535             subnets.append(subnet)
 536         net["subnets"] = subnets
 537         net["encapsulation"] = net.get('provider:network_type')
 538         net["segmentation_id"] = net.get('provider:segmentation_id')
 539         return net
 540
 541     def delete_network(self, net_id):
 542         '''Deletes a tenant network from VIM. Returns the old network identifier'''
 543         self.logger.debug("Deleting network '%s' from VIM", net_id)
 544         try:
 545             self._reload_connection()
 546             #delete VM ports attached to this networks before the network
 547             ports = self.neutron.list_ports(network_id=net_id)
 548             for p in ports['ports']:
 549                 try:
 550                     self.neutron.delete_port(p["id"])
 551                 except Exception as e:
 552                     self.logger.error("Error deleting port %s: %s", p["id"], str(e))
 553             self.neutron.delete_network(net_id)
 554             return net_id
 555         except (neExceptions.ConnectionFailed, neExceptions.NetworkNotFoundClient, neExceptions.NeutronException,
 556                 ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 557             self._format_exception(e)
 558
 559     def refresh_nets_status(self, net_list):
 560         '''Get the status of the networks
 561            Params: the list of network identifiers
 562            Returns a dictionary with:
 563                 net_id:         #VIM id of this network
 564                     status:     #Mandatory. Text with one of:
 565                                 #  DELETED (not found at vim)
 566                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 567                                 #  OTHER (Vim reported other status not understood)
 568                                 #  ERROR (VIM indicates an ERROR status)
 569                                 #  ACTIVE, INACTIVE, DOWN (admin down),
 570                                 #  BUILD (on building process)
 571                                 #
 572                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 573                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 574
 575         '''
 576         net_dict={}
 577         for net_id in net_list:
 578             net = {}
 579             try:
 580                 net_vim = self.get_network(net_id)
 581                 if net_vim['status'] in netStatus2manoFormat:
 582                     net["status"] = netStatus2manoFormat[ net_vim['status'] ]
 583                 else:
 584                     net["status"] = "OTHER"
 585                     net["error_msg"] = "VIM status reported " + net_vim['status']
 586
 587                 if net['status'] == "ACTIVE" and not net_vim['admin_state_up']:
 588                     net['status'] = 'DOWN'
 589                 try:
 590                     net['vim_info'] = yaml.safe_dump(net_vim, default_flow_style=True, width=256)
 591                 except yaml.representer.RepresenterError:
 592                     net['vim_info'] = str(net_vim)
 593                 if net_vim.get('fault'):  #TODO
 594                     net['error_msg'] = str(net_vim['fault'])
 595             except vimconn.vimconnNotFoundException as e:
 596                 self.logger.error("Exception getting net status: %s", str(e))
 597                 net['status'] = "DELETED"
 598                 net['error_msg'] = str(e)
 599             except vimconn.vimconnException as e:
 600                 self.logger.error("Exception getting net status: %s", str(e))
 601                 net['status'] = "VIM_ERROR"
 602                 net['error_msg'] = str(e)
 603             net_dict[net_id] = net
 604         return net_dict
 605
 606     def get_flavor(self, flavor_id):
 607         '''Obtain flavor details from the  VIM. Returns the flavor dict details'''
 608         self.logger.debug("Getting flavor '%s'", flavor_id)
 609         try:
 610             self._reload_connection()
 611             flavor = self.nova.flavors.find(id=flavor_id)
 612             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 613             return flavor.to_dict()
 614         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 615             self._format_exception(e)
 616
 617     def get_flavor_id_from_data(self, flavor_dict):
 618         """Obtain flavor id that match the flavor description
 619            Returns the flavor_id or raises a vimconnNotFoundException
 620            flavor_dict: contains the required ram, vcpus, disk
 621            If 'use_existing_flavors' is set to True at config, the closer flavor that provides same or more ram, vcpus
 622                 and disk is returned. Otherwise a flavor with exactly same ram, vcpus and disk is returned or a
 623                 vimconnNotFoundException is raised
 624         """
 625         exact_match = False if self.config.get('use_existing_flavors') else True
 626         try:
 627             self._reload_connection()
 628             flavor_candidate_id = None
 629             flavor_candidate_data = (10000, 10000, 10000)
 630             flavor_target = (flavor_dict["ram"], flavor_dict["vcpus"], flavor_dict["disk"])
 631             # numa=None
 632             numas = flavor_dict.get("extended", {}).get("numas")
 633             if numas:
 634                 #TODO
 635                 raise vimconn.vimconnNotFoundException("Flavor with EPA still not implemted")
 636                 # if len(numas) > 1:
 637                 #     raise vimconn.vimconnNotFoundException("Cannot find any flavor with more than one numa")
 638                 # numa=numas[0]
 639                 # numas = extended.get("numas")
 640             for flavor in self.nova.flavors.list():
 641                 epa = flavor.get_keys()
 642                 if epa:
 643                     continue
 644                     # TODO
 645                 flavor_data = (flavor.ram, flavor.vcpus, flavor.disk)
 646                 if flavor_data == flavor_target:
 647                     return flavor.id
 648                 elif not exact_match and flavor_target < flavor_data < flavor_candidate_data:
 649                     flavor_candidate_id = flavor.id
 650                     flavor_candidate_data = flavor_data
 651             if not exact_match and flavor_candidate_id:
 652                 return flavor_candidate_id
 653             raise vimconn.vimconnNotFoundException("Cannot find any flavor matching '{}'".format(str(flavor_dict)))
 654         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 655             self._format_exception(e)
 656
 657     def new_flavor(self, flavor_data, change_name_if_used=True):
 658         '''Adds a tenant flavor to openstack VIM
 659         if change_name_if_used is True, it will change name in case of conflict, because it is not supported name repetition
 660         Returns the flavor identifier
 661         '''
 662         self.logger.debug("Adding flavor '%s'", str(flavor_data))
 663         retry=0
 664         max_retries=3
 665         name_suffix = 0
 666         name=flavor_data['name']
 667         while retry<max_retries:
 668             retry+=1
 669             try:
 670                 self._reload_connection()
 671                 if change_name_if_used:
 672                     #get used names
 673                     fl_names=[]
 674                     fl=self.nova.flavors.list()
 675                     for f in fl:
 676                         fl_names.append(f.name)
 677                     while name in fl_names:
 678                         name_suffix += 1
 679                         name = flavor_data['name']+"-" + str(name_suffix)
 680
 681                 ram = flavor_data.get('ram',64)
 682                 vcpus = flavor_data.get('vcpus',1)
 683                 numa_properties=None
 684
 685                 extended = flavor_data.get("extended")
 686                 if extended:
 687                     numas=extended.get("numas")
 688                     if numas:
 689                         numa_nodes = len(numas)
 690                         if numa_nodes > 1:
 691                             return -1, "Can not add flavor with more than one numa"
 692                         numa_properties = {"hw:numa_nodes":str(numa_nodes)}
 693                         numa_properties["hw:mem_page_size"] = "large"
 694                         numa_properties["hw:cpu_policy"] = "dedicated"
 695                         numa_properties["hw:numa_mempolicy"] = "strict"
 696                         if self.vim_type == "VIO":
 697                             numa_properties["vmware:extra_config"] = '{"numa.nodeAffinity":"0"}'
 698                             numa_properties["vmware:latency_sensitivity_level"] = "high"
 699                         for numa in numas:
 700                             #overwrite ram and vcpus
 701                             #check if key 'memory' is present in numa else use ram value at flavor
 702                             if 'memory' in numa:
 703                                 ram = numa['memory']*1024
 704                             #See for reference: https://specs.openstack.org/openstack/nova-specs/specs/mitaka/implemented/virt-driver-cpu-thread-pinning.html
 705                             if 'paired-threads' in numa:
 706                                 vcpus = numa['paired-threads']*2
 707                                 #cpu_thread_policy "require" implies that the compute node must have an STM architecture
 708                                 numa_properties["hw:cpu_thread_policy"] = "require"
 709                                 numa_properties["hw:cpu_policy"] = "dedicated"
 710                             elif 'cores' in numa:
 711                                 vcpus = numa['cores']
 712                                 # cpu_thread_policy "prefer" implies that the host must not have an SMT architecture, or a non-SMT architecture will be emulated
 713                                 numa_properties["hw:cpu_thread_policy"] = "isolate"
 714                                 numa_properties["hw:cpu_policy"] = "dedicated"
 715                             elif 'threads' in numa:
 716                                 vcpus = numa['threads']
 717                                 # cpu_thread_policy "prefer" implies that the host may or may not have an SMT architecture
 718                                 numa_properties["hw:cpu_thread_policy"] = "prefer"
 719                                 numa_properties["hw:cpu_policy"] = "dedicated"
 720                             # for interface in numa.get("interfaces",() ):
 721                             #     if interface["dedicated"]=="yes":
 722                             #         raise vimconn.vimconnException("Passthrough interfaces are not supported for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable)
 723                             #     #TODO, add the key 'pci_passthrough:alias"="<label at config>:<number ifaces>"' when a way to connect it is available
 724
 725                 #create flavor
 726                 new_flavor=self.nova.flavors.create(name,
 727                                 ram,
 728                                 vcpus,
 729                                 flavor_data.get('disk',0),
 730                                 is_public=flavor_data.get('is_public', True)
 731                             )
 732                 #add metadata
 733                 if numa_properties:
 734                     new_flavor.set_keys(numa_properties)
 735                 return new_flavor.id
 736             except nvExceptions.Conflict as e:
 737                 if change_name_if_used and retry < max_retries:
 738                     continue
 739                 self._format_exception(e)
 740             #except nvExceptions.BadRequest as e:
 741             except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 742                 self._format_exception(e)
 743
 744     def delete_flavor(self,flavor_id):
 745         '''Deletes a tenant flavor from openstack VIM. Returns the old flavor_id
 746         '''
 747         try:
 748             self._reload_connection()
 749             self.nova.flavors.delete(flavor_id)
 750             return flavor_id
 751         #except nvExceptions.BadRequest as e:
 752         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 753             self._format_exception(e)
 754
 755     def new_image(self,image_dict):
 756         '''
 757         Adds a tenant image to VIM. imge_dict is a dictionary with:
 758             name: name
 759             disk_format: qcow2, vhd, vmdk, raw (by default), ...
 760             location: path or URI
 761             public: "yes" or "no"
 762             metadata: metadata of the image
 763         Returns the image_id
 764         '''
 765         retry=0
 766         max_retries=3
 767         while retry<max_retries:
 768             retry+=1
 769             try:
 770                 self._reload_connection()
 771                 #determine format  http://docs.openstack.org/developer/glance/formats.html
 772                 if "disk_format" in image_dict:
 773                     disk_format=image_dict["disk_format"]
 774                 else: #autodiscover based on extension
 775                     if image_dict['location'][-6:]==".qcow2":
 776                         disk_format="qcow2"
 777                     elif image_dict['location'][-4:]==".vhd":
 778                         disk_format="vhd"
 779                     elif image_dict['location'][-5:]==".vmdk":
 780                         disk_format="vmdk"
 781                     elif image_dict['location'][-4:]==".vdi":
 782                         disk_format="vdi"
 783                     elif image_dict['location'][-4:]==".iso":
 784                         disk_format="iso"
 785                     elif image_dict['location'][-4:]==".aki":
 786                         disk_format="aki"
 787                     elif image_dict['location'][-4:]==".ari":
 788                         disk_format="ari"
 789                     elif image_dict['location'][-4:]==".ami":
 790                         disk_format="ami"
 791                     else:
 792                         disk_format="raw"
 793                 self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location'])
 794                 if image_dict['location'][0:4]=="http":
 795                     new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 796                             container_format="bare", location=image_dict['location'], disk_format=disk_format)
 797                 else: #local path
 798                     with open(image_dict['location']) as fimage:
 799                         new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 800                             container_format="bare", data=fimage, disk_format=disk_format)
 801                 #insert metadata. We cannot use 'new_image.properties.setdefault'
 802                 #because nova and glance are "INDEPENDENT" and we are using nova for reading metadata
 803                 new_image_nova=self.nova.images.find(id=new_image.id)
 804                 new_image_nova.metadata.setdefault('location',image_dict['location'])
 805                 metadata_to_load = image_dict.get('metadata')
 806                 if metadata_to_load:
 807                     for k,v in yaml.load(metadata_to_load).iteritems():
 808                         new_image_nova.metadata.setdefault(k,v)
 809                 return new_image.id
 810             except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e:
 811                 self._format_exception(e)
 812             except (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 813                 if retry==max_retries:
 814                     continue
 815                 self._format_exception(e)
 816             except IOError as e:  #can not open the file
 817                 raise vimconn.vimconnConnectionException(type(e).__name__ + ": " + str(e)+ " for " + image_dict['location'],
 818                                                          http_code=vimconn.HTTP_Bad_Request)
 819
 820     def delete_image(self, image_id):
 821         '''Deletes a tenant image from openstack VIM. Returns the old id
 822         '''
 823         try:
 824             self._reload_connection()
 825             self.nova.images.delete(image_id)
 826             return image_id
 827         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e: #TODO remove
 828             self._format_exception(e)
 829
 830     def get_image_id_from_path(self, path):
 831         '''Get the image id from image path in the VIM database. Returns the image_id'''
 832         try:
 833             self._reload_connection()
 834             images = self.nova.images.list()
 835             for image in images:
 836                 if image.metadata.get("location")==path:
 837                     return image.id
 838             raise vimconn.vimconnNotFoundException("image with location '{}' not found".format( path))
 839         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 840             self._format_exception(e)
 841
 842     def get_image_list(self, filter_dict={}):
 843         '''Obtain tenant images from VIM
 844         Filter_dict can be:
 845             id: image id
 846             name: image name
 847             checksum: image checksum
 848         Returns the image list of dictionaries:
 849             [{<the fields at Filter_dict plus some VIM specific>}, ...]
 850             List can be empty
 851         '''
 852         self.logger.debug("Getting image list from VIM filter: '%s'", str(filter_dict))
 853         try:
 854             self._reload_connection()
 855             filter_dict_os = filter_dict.copy()
 856             #First we filter by the available filter fields: name, id. The others are removed.
 857             filter_dict_os.pop('checksum', None)
 858             image_list = self.nova.images.findall(**filter_dict_os)
 859             if len(image_list) == 0:
 860                 return []
 861             #Then we filter by the rest of filter fields: checksum
 862             filtered_list = []
 863             for image in image_list:
 864                 try:
 865                     image_class = self.glance.images.get(image.id)
 866                     if 'checksum' not in filter_dict or image_class['checksum'] == filter_dict.get('checksum'):
 867                         filtered_list.append(image_class.copy())
 868                 except gl1Exceptions.HTTPNotFound:
 869                     pass
 870             return filtered_list
 871         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 872             self._format_exception(e)
 873
 874     def __wait_for_vm(self, vm_id, status):
 875         """wait until vm is in the desired status and return True.
 876         If the VM gets in ERROR status, return false.
 877         If the timeout is reached generate an exception"""
 878         elapsed_time = 0
 879         while elapsed_time < server_timeout:
 880             vm_status = self.nova.servers.get(vm_id).status
 881             if vm_status == status:
 882                 return True
 883             if vm_status == 'ERROR':
 884                 return False
 885             time.sleep(5)
 886             elapsed_time += 5
 887
 888         # if we exceeded the timeout rollback
 889         if elapsed_time >= server_timeout:
 890             raise vimconn.vimconnException('Timeout waiting for instance ' + vm_id + ' to get ' + status,
 891                                            http_code=vimconn.HTTP_Request_Timeout)
 892
 893     def _get_openstack_availablity_zones(self):
 894         """
 895         Get from openstack availability zones available
 896         :return:
 897         """
 898         try:
 899             openstack_availability_zone = self.nova.availability_zones.list()
 900             openstack_availability_zone = [str(zone.zoneName) for zone in openstack_availability_zone
 901                                            if zone.zoneName != 'internal']
 902             return openstack_availability_zone
 903         except Exception as e:
 904             return None
 905
 906     def _set_availablity_zones(self):
 907         """
 908         Set vim availablity zone
 909         :return:
 910         """
 911
 912         if 'availability_zone' in self.config:
 913             vim_availability_zones = self.config.get('availability_zone')
 914             if isinstance(vim_availability_zones, str):
 915                 self.availability_zone = [vim_availability_zones]
 916             elif isinstance(vim_availability_zones, list):
 917                 self.availability_zone = vim_availability_zones
 918         else:
 919             self.availability_zone = self._get_openstack_availablity_zones()
 920
 921     def _get_vm_availability_zone(self, availability_zone_index, availability_zone_list):
 922         """
 923         Return thge availability zone to be used by the created VM.
 924         :return: The VIM availability zone to be used or None
 925         """
 926         if availability_zone_index is None:
 927             if not self.config.get('availability_zone'):
 928                 return None
 929             elif isinstance(self.config.get('availability_zone'), str):
 930                 return self.config['availability_zone']
 931             else:
 932                 # TODO consider using a different parameter at config for default AV and AV list match
 933                 return self.config['availability_zone'][0]
 934
 935         vim_availability_zones = self.availability_zone
 936         # check if VIM offer enough availability zones describe in the VNFD
 937         if vim_availability_zones and len(availability_zone_list) <= len(vim_availability_zones):
 938             # check if all the names of NFV AV match VIM AV names
 939             match_by_index = False
 940             for av in availability_zone_list:
 941                 if av not in vim_availability_zones:
 942                     match_by_index = True
 943                     break
 944             if match_by_index:
 945                 return vim_availability_zones[availability_zone_index]
 946             else:
 947                 return availability_zone_list[availability_zone_index]
 948         else:
 949             raise vimconn.vimconnConflictException("No enough availability zones at VIM for this deployment")
 950
 951     def new_vminstance(self, name, description, start, image_id, flavor_id, net_list, cloud_config=None, disk_list=None,
 952                        availability_zone_index=None, availability_zone_list=None):
 953         """Adds a VM instance to VIM
 954         Params:
 955             start: indicates if VM must start or boot in pause mode. Ignored
 956             image_id,flavor_id: iamge and flavor uuid
 957             net_list: list of interfaces, each one is a dictionary with:
 958                 name:
 959                 net_id: network uuid to connect
 960                 vpci: virtual vcpi to assign, ignored because openstack lack #TODO
 961                 model: interface model, ignored #TODO
 962                 mac_address: used for  SR-IOV ifaces #TODO for other types
 963                 use: 'data', 'bridge',  'mgmt'
 964                 type: 'virtual', 'PCI-PASSTHROUGH'('PF'), 'SR-IOV'('VF'), 'VFnotShared'
 965                 vim_id: filled/added by this function
 966                 floating_ip: True/False (or it can be None)
 967             'cloud_config': (optional) dictionary with:
 968             'key-pairs': (optional) list of strings with the public key to be inserted to the default user
 969             'users': (optional) list of users to be inserted, each item is a dict with:
 970                 'name': (mandatory) user name,
 971                 'key-pairs': (optional) list of strings with the public key to be inserted to the user
 972             'user-data': (optional) string is a text script to be passed directly to cloud-init
 973             'config-files': (optional). List of files to be transferred. Each item is a dict with:
 974                 'dest': (mandatory) string with the destination absolute path
 975                 'encoding': (optional, by default text). Can be one of:
 976                     'b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64'
 977                 'content' (mandatory): string with the content of the file
 978                 'permissions': (optional) string with file permissions, typically octal notation '0644'
 979                 'owner': (optional) file owner, string with the format 'owner:group'
 980             'boot-data-drive': boolean to indicate if user-data must be passed using a boot drive (hard disk)
 981             'disk_list': (optional) list with additional disks to the VM. Each item is a dict with:
 982                 'image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted
 983                 'size': (mandatory) string with the size of the disk in GB
 984                 'vim_id' (optional) should use this existing volume id
 985             availability_zone_index: Index of availability_zone_list to use for this this VM. None if not AV required
 986             availability_zone_list: list of availability zones given by user in the VNFD descriptor.  Ignore if
 987                 availability_zone_index is None
 988                 #TODO ip, security groups
 989         Returns a tuple with the instance identifier and created_items or raises an exception on error
 990             created_items can be None or a dictionary where this method can include key-values that will be passed to
 991             the method delete_vminstance and action_vminstance. Can be used to store created ports, volumes, etc.
 992             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
 993             as not present.
 994         """
 995         self.logger.debug("new_vminstance input: image='%s' flavor='%s' nics='%s'",image_id, flavor_id,str(net_list))
 996         try:
 997             server = None
 998             created_items = {}
 999             # metadata = {}
1000             net_list_vim = []
1001             external_network = []   # list of external networks to be connected to instance, later on used to create floating_ip
1002             no_secured_ports = []   # List of port-is with port-security disabled
1003             self._reload_connection()
1004             # metadata_vpci = {}   # For a specific neutron plugin
1005             block_device_mapping = None
1006             for net in net_list:
1007                 if not net.get("net_id"):   # skip non connected iface
1008                     continue
1009
1010                 port_dict={
1011                     "network_id": net["net_id"],
1012                     "name": net.get("name"),
1013                     "admin_state_up": True
1014                 }
1015                 if net["type"]=="virtual":
1016                     pass
1017                     # if "vpci" in net:
1018                     #     metadata_vpci[ net["net_id"] ] = [[ net["vpci"], "" ]]
1019                 elif net["type"] == "VF" or net["type"] == "SR-IOV":  # for VF
1020                     # if "vpci" in net:
1021                     #     if "VF" not in metadata_vpci:
1022                     #         metadata_vpci["VF"]=[]
1023                     #     metadata_vpci["VF"].append([ net["vpci"], "" ])
1024                     port_dict["binding:vnic_type"]="direct"
1025                     # VIO specific Changes
1026                     if self.vim_type == "VIO":
1027                         # Need to create port with port_security_enabled = False and no-security-groups
1028                         port_dict["port_security_enabled"]=False
1029                         port_dict["provider_security_groups"]=[]
1030                         port_dict["security_groups"]=[]
1031                 else:   # For PT PCI-PASSTHROUGH
1032                     # VIO specific Changes
1033                     # Current VIO release does not support port with type 'direct-physical'
1034                     # So no need to create virtual port in case of PCI-device.
1035                     # Will update port_dict code when support gets added in next VIO release
1036                     if self.vim_type == "VIO":
1037                         raise vimconn.vimconnNotSupportedException(
1038                             "Current VIO release does not support full passthrough (PT)")
1039                     # if "vpci" in net:
1040                     #     if "PF" not in metadata_vpci:
1041                     #         metadata_vpci["PF"]=[]
1042                     #     metadata_vpci["PF"].append([ net["vpci"], "" ])
1043                     port_dict["binding:vnic_type"]="direct-physical"
1044                 if not port_dict["name"]:
1045                     port_dict["name"]=name
1046                 if net.get("mac_address"):
1047                     port_dict["mac_address"]=net["mac_address"]
1048                 if net.get("ip_address"):
1049                     port_dict["fixed_ips"] = [{'ip_address': net["ip_address"]}]
1050                     # TODO add 'subnet_id': <subnet_id>
1051                 new_port = self.neutron.create_port({"port": port_dict })
1052                 created_items["port:" + str(new_port["port"]["id"])] = True
1053                 net["mac_adress"] = new_port["port"]["mac_address"]
1054                 net["vim_id"] = new_port["port"]["id"]
1055                 # if try to use a network without subnetwork, it will return a emtpy list
1056                 fixed_ips = new_port["port"].get("fixed_ips")
1057                 if fixed_ips:
1058                     net["ip"] = fixed_ips[0].get("ip_address")
1059                 else:
1060                     net["ip"] = None
1061
1062                 port = {"port-id": new_port["port"]["id"]}
1063                 if float(self.nova.api_version.get_string()) >= 2.32:
1064                     port["tag"] = new_port["port"]["name"]
1065                 net_list_vim.append(port)
1066
1067                 if net.get('floating_ip', False):
1068                     net['exit_on_floating_ip_error'] = True
1069                     external_network.append(net)
1070                 elif net['use'] == 'mgmt' and self.config.get('use_floating_ip'):
1071                     net['exit_on_floating_ip_error'] = False
1072                     external_network.append(net)
1073                     net['floating_ip'] = self.config.get('use_floating_ip')
1074
1075                 # If port security is disabled when the port has not yet been attached to the VM, then all vm traffic is dropped.
1076                 # As a workaround we wait until the VM is active and then disable the port-security
1077                 if net.get("port_security") == False and not self.config.get("no_port_security_extension"):
1078                     no_secured_ports.append(new_port["port"]["id"])
1079
1080             # if metadata_vpci:
1081             #     metadata = {"pci_assignement": json.dumps(metadata_vpci)}
1082             #     if len(metadata["pci_assignement"]) >255:
1083             #         #limit the metadata size
1084             #         #metadata["pci_assignement"] = metadata["pci_assignement"][0:255]
1085             #         self.logger.warn("Metadata deleted since it exceeds the expected length (255) ")
1086             #         metadata = {}
1087
1088             self.logger.debug("name '%s' image_id '%s'flavor_id '%s' net_list_vim '%s' description '%s'",
1089                               name, image_id, flavor_id, str(net_list_vim), description)
1090
1091             security_groups = self.config.get('security_groups')
1092             if type(security_groups) is str:
1093                 security_groups = ( security_groups, )
1094             # cloud config
1095             config_drive, userdata = self._create_user_data(cloud_config)
1096
1097             # Create additional volumes in case these are present in disk_list
1098             base_disk_index = ord('b')
1099             if disk_list:
1100                 block_device_mapping = {}
1101                 for disk in disk_list:
1102                     if disk.get('vim_id'):
1103                         block_device_mapping['_vd' + chr(base_disk_index)] = disk['vim_id']
1104                     else:
1105                         if 'image_id' in disk:
1106                             volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1107                                                                 chr(base_disk_index), imageRef=disk['image_id'])
1108                         else:
1109                             volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1110                                                                 chr(base_disk_index))
1111                         created_items["volume:" + str(volume.id)] = True
1112                         block_device_mapping['_vd' + chr(base_disk_index)] = volume.id
1113                     base_disk_index += 1
1114
1115                 # Wait until created volumes are with status available
1116                 elapsed_time = 0
1117                 while elapsed_time < volume_timeout:
1118                     for created_item in created_items:
1119                         v, _, volume_id = created_item.partition(":")
1120                         if v == 'volume':
1121                             if self.cinder.volumes.get(volume_id).status != 'available':
1122                                 break
1123                     else:  # all ready: break from while
1124                         break
1125                     time.sleep(5)
1126                     elapsed_time += 5
1127                 # If we exceeded the timeout rollback
1128                 if elapsed_time >= volume_timeout:
1129                     raise vimconn.vimconnException('Timeout creating volumes for instance ' + name,
1130                                                    http_code=vimconn.HTTP_Request_Timeout)
1131             # get availability Zone
1132             vm_av_zone = self._get_vm_availability_zone(availability_zone_index, availability_zone_list)
1133
1134             self.logger.debug("nova.servers.create({}, {}, {}, nics={}, security_groups={}, "
1135                               "availability_zone={}, key_name={}, userdata={}, config_drive={}, "
1136                               "block_device_mapping={})".format(name, image_id, flavor_id, net_list_vim,
1137                                                                 security_groups, vm_av_zone, self.config.get('keypair'),
1138                                                                 userdata, config_drive, block_device_mapping))
1139             server = self.nova.servers.create(name, image_id, flavor_id, nics=net_list_vim,
1140                                               security_groups=security_groups,
1141                                               availability_zone=vm_av_zone,
1142                                               key_name=self.config.get('keypair'),
1143                                               userdata=userdata,
1144                                               config_drive=config_drive,
1145                                               block_device_mapping=block_device_mapping
1146                                               )  # , description=description)
1147
1148             vm_start_time = time.time()
1149             # Previously mentioned workaround to wait until the VM is active and then disable the port-security
1150             if no_secured_ports:
1151                 self.__wait_for_vm(server.id, 'ACTIVE')
1152
1153             for port_id in no_secured_ports:
1154                 try:
1155                     self.neutron.update_port(port_id,
1156                                              {"port": {"port_security_enabled": False, "security_groups": None}})
1157                 except Exception as e:
1158                     raise vimconn.vimconnException("It was not possible to disable port security for port {}".format(
1159                         port_id))
1160             # print "DONE :-)", server
1161
1162             # pool_id = None
1163             if external_network:
1164                 floating_ips = self.neutron.list_floatingips().get("floatingips", ())
1165             for floating_network in external_network:
1166                 try:
1167                     assigned = False
1168                     while not assigned:
1169                         if floating_ips:
1170                             ip = floating_ips.pop(0)
1171                             if ip.get("port_id", False) or ip.get('tenant_id') != server.tenant_id:
1172                                 continue
1173                             if isinstance(floating_network['floating_ip'], str):
1174                                 if ip.get("floating_network_id") != floating_network['floating_ip']:
1175                                     continue
1176                             free_floating_ip = ip.get("floating_ip_address")
1177                         else:
1178                             if isinstance(floating_network['floating_ip'], str) and \
1179                                 floating_network['floating_ip'].lower() != "true":
1180                                 pool_id = floating_network['floating_ip']
1181                             else:
1182                                 # Find the external network
1183                                 external_nets = list()
1184                                 for net in self.neutron.list_networks()['networks']:
1185                                     if net['router:external']:
1186                                             external_nets.append(net)
1187
1188                                 if len(external_nets) == 0:
1189                                     raise vimconn.vimconnException("Cannot create floating_ip automatically since no external "
1190                                                                    "network is present",
1191                                                                     http_code=vimconn.HTTP_Conflict)
1192                                 if len(external_nets) > 1:
1193                                     raise vimconn.vimconnException("Cannot create floating_ip automatically since multiple "
1194                                                                    "external networks are present",
1195                                                                    http_code=vimconn.HTTP_Conflict)
1196
1197                                 pool_id = external_nets[0].get('id')
1198                             param = {'floatingip': {'floating_network_id': pool_id, 'tenant_id': server.tenant_id}}
1199                             try:
1200                                 # self.logger.debug("Creating floating IP")
1201                                 new_floating_ip = self.neutron.create_floatingip(param)
1202                                 free_floating_ip = new_floating_ip['floatingip']['floating_ip_address']
1203                             except Exception as e:
1204                                 raise vimconn.vimconnException(type(e).__name__ + ": Cannot create new floating_ip " +
1205                                                                str(e), http_code=vimconn.HTTP_Conflict)
1206
1207                         fix_ip = floating_network.get('ip')
1208                         while not assigned:
1209                             try:
1210                                 server.add_floating_ip(free_floating_ip, fix_ip)
1211                                 assigned = True
1212                             except Exception as e:
1213                                 # openstack need some time after VM creation to asign an IP. So retry if fails
1214                                 vm_status = self.nova.servers.get(server.id).status
1215                                 if vm_status != 'ACTIVE' and vm_status != 'ERROR':
1216                                     if time.time() - vm_start_time < server_timeout:
1217                                         time.sleep(5)
1218                                         continue
1219                                 raise vimconn.vimconnException(
1220                                     "Cannot create floating_ip: {} {}".format(type(e).__name__, e),
1221                                     http_code=vimconn.HTTP_Conflict)
1222
1223                 except Exception as e:
1224                     if not floating_network['exit_on_floating_ip_error']:
1225                         self.logger.warn("Cannot create floating_ip. %s", str(e))
1226                         continue
1227                     raise
1228
1229             return server.id, created_items
1230 #        except nvExceptions.NotFound as e:
1231 #            error_value=-vimconn.HTTP_Not_Found
1232 #            error_text= "vm instance %s not found" % vm_id
1233 #        except TypeError as e:
1234 #            raise vimconn.vimconnException(type(e).__name__ + ": "+  str(e), http_code=vimconn.HTTP_Bad_Request)
1235
1236         except Exception as e:
1237             server_id = None
1238             if server:
1239                 server_id = server.id
1240             try:
1241                 self.delete_vminstance(server_id, created_items)
1242             except Exception as e2:
1243                 self.logger.error("new_vminstance rollback fail {}".format(e2))
1244
1245             self._format_exception(e)
1246
1247     def get_vminstance(self,vm_id):
1248         '''Returns the VM instance information from VIM'''
1249         #self.logger.debug("Getting VM from VIM")
1250         try:
1251             self._reload_connection()
1252             server = self.nova.servers.find(id=vm_id)
1253             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
1254             return server.to_dict()
1255         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1256             self._format_exception(e)
1257
1258     def get_vminstance_console(self,vm_id, console_type="vnc"):
1259         '''
1260         Get a console for the virtual machine
1261         Params:
1262             vm_id: uuid of the VM
1263             console_type, can be:
1264                 "novnc" (by default), "xvpvnc" for VNC types,
1265                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
1266         Returns dict with the console parameters:
1267                 protocol: ssh, ftp, http, https, ...
1268                 server:   usually ip address
1269                 port:     the http, ssh, ... port
1270                 suffix:   extra text, e.g. the http path and query string
1271         '''
1272         self.logger.debug("Getting VM CONSOLE from VIM")
1273         try:
1274             self._reload_connection()
1275             server = self.nova.servers.find(id=vm_id)
1276             if console_type == None or console_type == "novnc":
1277                 console_dict = server.get_vnc_console("novnc")
1278             elif console_type == "xvpvnc":
1279                 console_dict = server.get_vnc_console(console_type)
1280             elif console_type == "rdp-html5":
1281                 console_dict = server.get_rdp_console(console_type)
1282             elif console_type == "spice-html5":
1283                 console_dict = server.get_spice_console(console_type)
1284             else:
1285                 raise vimconn.vimconnException("console type '{}' not allowed".format(console_type), http_code=vimconn.HTTP_Bad_Request)
1286
1287             console_dict1 = console_dict.get("console")
1288             if console_dict1:
1289                 console_url = console_dict1.get("url")
1290                 if console_url:
1291                     #parse console_url
1292                     protocol_index = console_url.find("//")
1293                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1294                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1295                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1296                         return -vimconn.HTTP_Internal_Server_Error, "Unexpected response from VIM"
1297                     console_dict={"protocol": console_url[0:protocol_index],
1298                                   "server":   console_url[protocol_index+2:port_index],
1299                                   "port":     console_url[port_index:suffix_index],
1300                                   "suffix":   console_url[suffix_index+1:]
1301                                   }
1302                     protocol_index += 2
1303                     return console_dict
1304             raise vimconn.vimconnUnexpectedResponse("Unexpected response from VIM")
1305
1306         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.BadRequest, ConnectionError) as e:
1307             self._format_exception(e)
1308
1309     def delete_vminstance(self, vm_id, created_items=None):
1310         '''Removes a VM instance from VIM. Returns the old identifier
1311         '''
1312         #print "osconnector: Getting VM from VIM"
1313         if created_items == None:
1314             created_items = {}
1315         try:
1316             self._reload_connection()
1317             # delete VM ports attached to this networks before the virtual machine
1318             for k, v in created_items.items():
1319                 if not v:  # skip already deleted
1320                     continue
1321                 try:
1322                     k_item, _, k_id = k.partition(":")
1323                     if k_item == "port":
1324                         self.neutron.delete_port(k_id)
1325                 except Exception as e:
1326                     self.logger.error("Error deleting port: {}: {}".format(type(e).__name__, e))
1327
1328             # #commented because detaching the volumes makes the servers.delete not work properly ?!?
1329             # #dettach volumes attached
1330             # server = self.nova.servers.get(vm_id)
1331             # volumes_attached_dict = server._info['os-extended-volumes:volumes_attached']   #volume['id']
1332             # #for volume in volumes_attached_dict:
1333             # #    self.cinder.volumes.detach(volume['id'])
1334
1335             if vm_id:
1336                 self.nova.servers.delete(vm_id)
1337
1338             # delete volumes. Although having detached, they should have in active status before deleting
1339             # we ensure in this loop
1340             keep_waiting = True
1341             elapsed_time = 0
1342             while keep_waiting and elapsed_time < volume_timeout:
1343                 keep_waiting = False
1344                 for k, v in created_items.items():
1345                     if not v:  # skip already deleted
1346                         continue
1347                     try:
1348                         k_item, _, k_id = k.partition(":")
1349                         if k_item == "volume":
1350                             if self.cinder.volumes.get(k_id).status != 'available':
1351                                 keep_waiting = True
1352                             else:
1353                                 self.cinder.volumes.delete(k_id)
1354                     except Exception as e:
1355                         self.logger.error("Error deleting volume: {}: {}".format(type(e).__name__, e))
1356                 if keep_waiting:
1357                     time.sleep(1)
1358                     elapsed_time += 1
1359             return None
1360         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
1361             self._format_exception(e)
1362
1363     def refresh_vms_status(self, vm_list):
1364         '''Get the status of the virtual machines and their interfaces/ports
1365            Params: the list of VM identifiers
1366            Returns a dictionary with:
1367                 vm_id:          #VIM id of this Virtual Machine
1368                     status:     #Mandatory. Text with one of:
1369                                 #  DELETED (not found at vim)
1370                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1371                                 #  OTHER (Vim reported other status not understood)
1372                                 #  ERROR (VIM indicates an ERROR status)
1373                                 #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
1374                                 #  CREATING (on building process), ERROR
1375                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
1376                                 #
1377                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
1378                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
1379                     interfaces:
1380                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
1381                         mac_address:      #Text format XX:XX:XX:XX:XX:XX
1382                         vim_net_id:       #network id where this interface is connected
1383                         vim_interface_id: #interface/port VIM id
1384                         ip_address:       #null, or text with IPv4, IPv6 address
1385                         compute_node:     #identification of compute node where PF,VF interface is allocated
1386                         pci:              #PCI address of the NIC that hosts the PF,VF
1387                         vlan:             #physical VLAN used for VF
1388         '''
1389         vm_dict={}
1390         self.logger.debug("refresh_vms status: Getting tenant VM instance information from VIM")
1391         for vm_id in vm_list:
1392             vm={}
1393             try:
1394                 vm_vim = self.get_vminstance(vm_id)
1395                 if vm_vim['status'] in vmStatus2manoFormat:
1396                     vm['status']    =  vmStatus2manoFormat[ vm_vim['status'] ]
1397                 else:
1398                     vm['status']    = "OTHER"
1399                     vm['error_msg'] = "VIM status reported " + vm_vim['status']
1400                 try:
1401                     vm['vim_info']  = yaml.safe_dump(vm_vim, default_flow_style=True, width=256)
1402                 except yaml.representer.RepresenterError:
1403                     vm['vim_info'] = str(vm_vim)
1404                 vm["interfaces"] = []
1405                 if vm_vim.get('fault'):
1406                     vm['error_msg'] = str(vm_vim['fault'])
1407                 #get interfaces
1408                 try:
1409                     self._reload_connection()
1410                     port_dict = self.neutron.list_ports(device_id=vm_id)
1411                     for port in port_dict["ports"]:
1412                         interface={}
1413                         try:
1414                             interface['vim_info'] = yaml.safe_dump(port, default_flow_style=True, width=256)
1415                         except yaml.representer.RepresenterError:
1416                             interface['vim_info'] = str(port)
1417                         interface["mac_address"] = port.get("mac_address")
1418                         interface["vim_net_id"] = port["network_id"]
1419                         interface["vim_interface_id"] = port["id"]
1420                         # check if OS-EXT-SRV-ATTR:host is there,
1421                         # in case of non-admin credentials, it will be missing
1422                         if vm_vim.get('OS-EXT-SRV-ATTR:host'):
1423                             interface["compute_node"] = vm_vim['OS-EXT-SRV-ATTR:host']
1424                         interface["pci"] = None
1425
1426                         # check if binding:profile is there,
1427                         # in case of non-admin credentials, it will be missing
1428                         if port.get('binding:profile'):
1429                             if port['binding:profile'].get('pci_slot'):
1430                                 # TODO: At the moment sr-iov pci addresses are converted to PF pci addresses by setting the slot to 0x00
1431                                 # TODO: This is just a workaround valid for niantinc. Find a better way to do so
1432                                 #   CHANGE DDDD:BB:SS.F to DDDD:BB:00.(F%2)   assuming there are 2 ports per nic
1433                                 pci = port['binding:profile']['pci_slot']
1434                                 # interface["pci"] = pci[:-4] + "00." + str(int(pci[-1]) % 2)
1435                                 interface["pci"] = pci
1436                         interface["vlan"] = None
1437                         #if network is of type vlan and port is of type direct (sr-iov) then set vlan id
1438                         network = self.neutron.show_network(port["network_id"])
1439                         if network['network'].get('provider:network_type') == 'vlan' and \
1440                             port.get("binding:vnic_type") == "direct":
1441                             interface["vlan"] = network['network'].get('provider:segmentation_id')
1442                         ips=[]
1443                         #look for floating ip address
1444                         try:
1445                             floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
1446                             if floating_ip_dict.get("floatingips"):
1447                                 ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address") )
1448                         except Exception:
1449                             pass
1450
1451                         for subnet in port["fixed_ips"]:
1452                             ips.append(subnet["ip_address"])
1453                         interface["ip_address"] = ";".join(ips)
1454                         vm["interfaces"].append(interface)
1455                 except Exception as e:
1456                     self.logger.error("Error getting vm interface information {}: {}".format(type(e).__name__, e),
1457                                       exc_info=True)
1458             except vimconn.vimconnNotFoundException as e:
1459                 self.logger.error("Exception getting vm status: %s", str(e))
1460                 vm['status'] = "DELETED"
1461                 vm['error_msg'] = str(e)
1462             except vimconn.vimconnException as e:
1463                 self.logger.error("Exception getting vm status: %s", str(e))
1464                 vm['status'] = "VIM_ERROR"
1465                 vm['error_msg'] = str(e)
1466             vm_dict[vm_id] = vm
1467         return vm_dict
1468
1469     def action_vminstance(self, vm_id, action_dict, created_items={}):
1470         '''Send and action over a VM instance from VIM
1471         Returns None or the console dict if the action was successfully sent to the VIM'''
1472         self.logger.debug("Action over VM '%s': %s", vm_id, str(action_dict))
1473         try:
1474             self._reload_connection()
1475             server = self.nova.servers.find(id=vm_id)
1476             if "start" in action_dict:
1477                 if action_dict["start"]=="rebuild":
1478                     server.rebuild()
1479                 else:
1480                     if server.status=="PAUSED":
1481                         server.unpause()
1482                     elif server.status=="SUSPENDED":
1483                         server.resume()
1484                     elif server.status=="SHUTOFF":
1485                         server.start()
1486             elif "pause" in action_dict:
1487                 server.pause()
1488             elif "resume" in action_dict:
1489                 server.resume()
1490             elif "shutoff" in action_dict or "shutdown" in action_dict:
1491                 server.stop()
1492             elif "forceOff" in action_dict:
1493                 server.stop() #TODO
1494             elif "terminate" in action_dict:
1495                 server.delete()
1496             elif "createImage" in action_dict:
1497                 server.create_image()
1498                 #"path":path_schema,
1499                 #"description":description_schema,
1500                 #"name":name_schema,
1501                 #"metadata":metadata_schema,
1502                 #"imageRef": id_schema,
1503                 #"disk": {"oneOf":[{"type": "null"}, {"type":"string"}] },
1504             elif "rebuild" in action_dict:
1505                 server.rebuild(server.image['id'])
1506             elif "reboot" in action_dict:
1507                 server.reboot() #reboot_type='SOFT'
1508             elif "console" in action_dict:
1509                 console_type = action_dict["console"]
1510                 if console_type == None or console_type == "novnc":
1511                     console_dict = server.get_vnc_console("novnc")
1512                 elif console_type == "xvpvnc":
1513                     console_dict = server.get_vnc_console(console_type)
1514                 elif console_type == "rdp-html5":
1515                     console_dict = server.get_rdp_console(console_type)
1516                 elif console_type == "spice-html5":
1517                     console_dict = server.get_spice_console(console_type)
1518                 else:
1519                     raise vimconn.vimconnException("console type '{}' not allowed".format(console_type),
1520                                                    http_code=vimconn.HTTP_Bad_Request)
1521                 try:
1522                     console_url = console_dict["console"]["url"]
1523                     #parse console_url
1524                     protocol_index = console_url.find("//")
1525                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1526                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1527                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1528                         raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1529                     console_dict2={"protocol": console_url[0:protocol_index],
1530                                   "server":   console_url[protocol_index+2 : port_index],
1531                                   "port":     int(console_url[port_index+1 : suffix_index]),
1532                                   "suffix":   console_url[suffix_index+1:]
1533                                   }
1534                     return console_dict2
1535                 except Exception as e:
1536                     raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1537
1538             return None
1539         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1540             self._format_exception(e)
1541         #TODO insert exception vimconn.HTTP_Unauthorized
1542
1543     ####### VIO Specific Changes #########
1544     def _genrate_vlanID(self):
1545         """
1546          Method to get unused vlanID
1547             Args:
1548                 None
1549             Returns:
1550                 vlanID
1551         """
1552         #Get used VLAN IDs
1553         usedVlanIDs = []
1554         networks = self.get_network_list()
1555         for net in networks:
1556             if net.get('provider:segmentation_id'):
1557                 usedVlanIDs.append(net.get('provider:segmentation_id'))
1558         used_vlanIDs = set(usedVlanIDs)
1559
1560         #find unused VLAN ID
1561         for vlanID_range in self.config.get('dataplane_net_vlan_range'):
1562             try:
1563                 start_vlanid , end_vlanid = map(int, vlanID_range.replace(" ", "").split("-"))
1564                 for vlanID in xrange(start_vlanid, end_vlanid + 1):
1565                     if vlanID not in used_vlanIDs:
1566                         return vlanID
1567             except Exception as exp:
1568                 raise vimconn.vimconnException("Exception {} occurred while generating VLAN ID.".format(exp))
1569         else:
1570             raise vimconn.vimconnConflictException("Unable to create the SRIOV VLAN network."\
1571                 " All given Vlan IDs {} are in use.".format(self.config.get('dataplane_net_vlan_range')))
1572
1573
1574     def _validate_vlan_ranges(self, dataplane_net_vlan_range):
1575         """
1576         Method to validate user given vlanID ranges
1577             Args:  None
1578             Returns: None
1579         """
1580         for vlanID_range in dataplane_net_vlan_range:
1581             vlan_range = vlanID_range.replace(" ", "")
1582             #validate format
1583             vlanID_pattern = r'(\d)*-(\d)*$'
1584             match_obj = re.match(vlanID_pattern, vlan_range)
1585             if not match_obj:
1586                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}.You must provide "\
1587                 "'dataplane_net_vlan_range' in format [start_ID - end_ID].".format(vlanID_range))
1588
1589             start_vlanid , end_vlanid = map(int,vlan_range.split("-"))
1590             if start_vlanid <= 0 :
1591                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1592                 "Start ID can not be zero. For VLAN "\
1593                 "networks valid IDs are 1 to 4094 ".format(vlanID_range))
1594             if end_vlanid > 4094 :
1595                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1596                 "End VLAN ID can not be greater than 4094. For VLAN "\
1597                 "networks valid IDs are 1 to 4094 ".format(vlanID_range))
1598
1599             if start_vlanid > end_vlanid:
1600                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1601                     "You must provide a 'dataplane_net_vlan_range' in format start_ID - end_ID and "\
1602                     "start_ID < end_ID ".format(vlanID_range))
1603
1604 #NOT USED FUNCTIONS
1605
1606     def new_external_port(self, port_data):
1607         #TODO openstack if needed
1608         '''Adds a external port to VIM'''
1609         '''Returns the port identifier'''
1610         return -vimconn.HTTP_Internal_Server_Error, "osconnector.new_external_port() not implemented"
1611
1612     def connect_port_network(self, port_id, network_id, admin=False):
1613         #TODO openstack if needed
1614         '''Connects a external port to a network'''
1615         '''Returns status code of the VIM response'''
1616         return -vimconn.HTTP_Internal_Server_Error, "osconnector.connect_port_network() not implemented"
1617
1618     def new_user(self, user_name, user_passwd, tenant_id=None):
1619         '''Adds a new user to openstack VIM'''
1620         '''Returns the user identifier'''
1621         self.logger.debug("osconnector: Adding a new user to VIM")
1622         try:
1623             self._reload_connection()
1624             user=self.keystone.users.create(user_name, user_passwd, tenant_id=tenant_id)
1625             #self.keystone.tenants.add_user(self.k_creds["username"], #role)
1626             return user.id
1627         except ksExceptions.ConnectionError as e:
1628             error_value=-vimconn.HTTP_Bad_Request
1629             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1630         except ksExceptions.ClientException as e: #TODO remove
1631             error_value=-vimconn.HTTP_Bad_Request
1632             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1633         #TODO insert exception vimconn.HTTP_Unauthorized
1634         #if reaching here is because an exception
1635         self.logger.debug("new_user " + error_text)
1636         return error_value, error_text
1637
1638     def delete_user(self, user_id):
1639         '''Delete a user from openstack VIM'''
1640         '''Returns the user identifier'''
1641         if self.debug:
1642             print("osconnector: Deleting  a  user from VIM")
1643         try:
1644             self._reload_connection()
1645             self.keystone.users.delete(user_id)
1646             return 1, user_id
1647         except ksExceptions.ConnectionError as e:
1648             error_value=-vimconn.HTTP_Bad_Request
1649             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1650         except ksExceptions.NotFound as e:
1651             error_value=-vimconn.HTTP_Not_Found
1652             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1653         except ksExceptions.ClientException as e: #TODO remove
1654             error_value=-vimconn.HTTP_Bad_Request
1655             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1656         #TODO insert exception vimconn.HTTP_Unauthorized
1657         #if reaching here is because an exception
1658             self.logger.debug("delete_tenant " + error_text)
1659         return error_value, error_text
1660
1661     def get_hosts_info(self):
1662         '''Get the information of deployed hosts
1663         Returns the hosts content'''
1664         if self.debug:
1665             print("osconnector: Getting Host info from VIM")
1666         try:
1667             h_list=[]
1668             self._reload_connection()
1669             hypervisors = self.nova.hypervisors.list()
1670             for hype in hypervisors:
1671                 h_list.append( hype.to_dict() )
1672             return 1, {"hosts":h_list}
1673         except nvExceptions.NotFound as e:
1674             error_value=-vimconn.HTTP_Not_Found
1675             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1676         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1677             error_value=-vimconn.HTTP_Bad_Request
1678             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1679         #TODO insert exception vimconn.HTTP_Unauthorized
1680         #if reaching here is because an exception
1681         self.logger.debug("get_hosts_info " + error_text)
1682         return error_value, error_text
1683
1684     def get_hosts(self, vim_tenant):
1685         '''Get the hosts and deployed instances
1686         Returns the hosts content'''
1687         r, hype_dict = self.get_hosts_info()
1688         if r<0:
1689             return r, hype_dict
1690         hypervisors = hype_dict["hosts"]
1691         try:
1692             servers = self.nova.servers.list()
1693             for hype in hypervisors:
1694                 for server in servers:
1695                     if server.to_dict()['OS-EXT-SRV-ATTR:hypervisor_hostname']==hype['hypervisor_hostname']:
1696                         if 'vm' in hype:
1697                             hype['vm'].append(server.id)
1698                         else:
1699                             hype['vm'] = [server.id]
1700             return 1, hype_dict
1701         except nvExceptions.NotFound as e:
1702             error_value=-vimconn.HTTP_Not_Found
1703             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1704         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1705             error_value=-vimconn.HTTP_Bad_Request
1706             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1707         #TODO insert exception vimconn.HTTP_Unauthorized
1708         #if reaching here is because an exception
1709         self.logger.debug("get_hosts " + error_text)
1710         return error_value, error_text
1711
1712     def new_classification(self, name, ctype, definition):
1713         self.logger.debug(
1714             'Adding a new (Traffic) Classification to VIM, named %s', name)
1715         try:
1716             new_class = None
1717             self._reload_connection()
1718             if ctype not in supportedClassificationTypes:
1719                 raise vimconn.vimconnNotSupportedException(
1720                         'OpenStack VIM connector doesn\'t support provided '
1721                         'Classification Type {}, supported ones are: '
1722                         '{}'.format(ctype, supportedClassificationTypes))
1723             if not self._validate_classification(ctype, definition):
1724                 raise vimconn.vimconnException(
1725                     'Incorrect Classification definition '
1726                     'for the type specified.')
1727             classification_dict = definition
1728             classification_dict['name'] = name
1729
1730             new_class = self.neutron.create_sfc_flow_classifier(
1731                 {'flow_classifier': classification_dict})
1732             return new_class['flow_classifier']['id']
1733         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1734                 neExceptions.NeutronException, ConnectionError) as e:
1735             self.logger.error(
1736                 'Creation of Classification failed.')
1737             self._format_exception(e)
1738
1739     def get_classification(self, class_id):
1740         self.logger.debug(" Getting Classification %s from VIM", class_id)
1741         filter_dict = {"id": class_id}
1742         class_list = self.get_classification_list(filter_dict)
1743         if len(class_list) == 0:
1744             raise vimconn.vimconnNotFoundException(
1745                 "Classification '{}' not found".format(class_id))
1746         elif len(class_list) > 1:
1747             raise vimconn.vimconnConflictException(
1748                 "Found more than one Classification with this criteria")
1749         classification = class_list[0]
1750         return classification
1751
1752     def get_classification_list(self, filter_dict={}):
1753         self.logger.debug("Getting Classifications from VIM filter: '%s'",
1754                           str(filter_dict))
1755         try:
1756             filter_dict_os = filter_dict.copy()
1757             self._reload_connection()
1758             if self.api_version3 and "tenant_id" in filter_dict_os:
1759                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1760             classification_dict = self.neutron.list_sfc_flow_classifiers(
1761                 **filter_dict_os)
1762             classification_list = classification_dict["flow_classifiers"]
1763             self.__classification_os2mano(classification_list)
1764             return classification_list
1765         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1766                 neExceptions.NeutronException, ConnectionError) as e:
1767             self._format_exception(e)
1768
1769     def delete_classification(self, class_id):
1770         self.logger.debug("Deleting Classification '%s' from VIM", class_id)
1771         try:
1772             self._reload_connection()
1773             self.neutron.delete_sfc_flow_classifier(class_id)
1774             return class_id
1775         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1776                 ksExceptions.ClientException, neExceptions.NeutronException,
1777                 ConnectionError) as e:
1778             self._format_exception(e)
1779
1780     def new_sfi(self, name, ingress_ports, egress_ports, sfc_encap=True):
1781         self.logger.debug(
1782             "Adding a new Service Function Instance to VIM, named '%s'", name)
1783         try:
1784             new_sfi = None
1785             self._reload_connection()
1786             correlation = None
1787             if sfc_encap:
1788                 correlation = 'nsh'
1789             if len(ingress_ports) != 1:
1790                 raise vimconn.vimconnNotSupportedException(
1791                     "OpenStack VIM connector can only have "
1792                     "1 ingress port per SFI")
1793             if len(egress_ports) != 1:
1794                 raise vimconn.vimconnNotSupportedException(
1795                     "OpenStack VIM connector can only have "
1796                     "1 egress port per SFI")
1797             sfi_dict = {'name': name,
1798                         'ingress': ingress_ports[0],
1799                         'egress': egress_ports[0],
1800                         'service_function_parameters': {
1801                             'correlation': correlation}}
1802             new_sfi = self.neutron.create_sfc_port_pair({'port_pair': sfi_dict})
1803             return new_sfi['port_pair']['id']
1804         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1805                 neExceptions.NeutronException, ConnectionError) as e:
1806             if new_sfi:
1807                 try:
1808                     self.neutron.delete_sfc_port_pair(
1809                         new_sfi['port_pair']['id'])
1810                 except Exception:
1811                     self.logger.error(
1812                         'Creation of Service Function Instance failed, with '
1813                         'subsequent deletion failure as well.')
1814             self._format_exception(e)
1815
1816     def get_sfi(self, sfi_id):
1817         self.logger.debug(
1818             'Getting Service Function Instance %s from VIM', sfi_id)
1819         filter_dict = {"id": sfi_id}
1820         sfi_list = self.get_sfi_list(filter_dict)
1821         if len(sfi_list) == 0:
1822             raise vimconn.vimconnNotFoundException(
1823                 "Service Function Instance '{}' not found".format(sfi_id))
1824         elif len(sfi_list) > 1:
1825             raise vimconn.vimconnConflictException(
1826                 'Found more than one Service Function Instance '
1827                 'with this criteria')
1828         sfi = sfi_list[0]
1829         return sfi
1830
1831     def get_sfi_list(self, filter_dict={}):
1832         self.logger.debug("Getting Service Function Instances from "
1833                           "VIM filter: '%s'", str(filter_dict))
1834         try:
1835             self._reload_connection()
1836             filter_dict_os = filter_dict.copy()
1837             if self.api_version3 and "tenant_id" in filter_dict_os:
1838                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1839             sfi_dict = self.neutron.list_sfc_port_pairs(**filter_dict_os)
1840             sfi_list = sfi_dict["port_pairs"]
1841             self.__sfi_os2mano(sfi_list)
1842             return sfi_list
1843         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1844                 neExceptions.NeutronException, ConnectionError) as e:
1845             self._format_exception(e)
1846
1847     def delete_sfi(self, sfi_id):
1848         self.logger.debug("Deleting Service Function Instance '%s' "
1849                           "from VIM", sfi_id)
1850         try:
1851             self._reload_connection()
1852             self.neutron.delete_sfc_port_pair(sfi_id)
1853             return sfi_id
1854         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1855                 ksExceptions.ClientException, neExceptions.NeutronException,
1856                 ConnectionError) as e:
1857             self._format_exception(e)
1858
1859     def new_sf(self, name, sfis, sfc_encap=True):
1860         self.logger.debug("Adding a new Service Function to VIM, "
1861                           "named '%s'", name)
1862         try:
1863             new_sf = None
1864             self._reload_connection()
1865             # correlation = None
1866             # if sfc_encap:
1867             #     correlation = 'nsh'
1868             for instance in sfis:
1869                 sfi = self.get_sfi(instance)
1870                 if sfi.get('sfc_encap') != sfc_encap:
1871                     raise vimconn.vimconnNotSupportedException(
1872                         "OpenStack VIM connector requires all SFIs of the "
1873                         "same SF to share the same SFC Encapsulation")
1874             sf_dict = {'name': name,
1875                        'port_pairs': sfis}
1876             new_sf = self.neutron.create_sfc_port_pair_group({
1877                 'port_pair_group': sf_dict})
1878             return new_sf['port_pair_group']['id']
1879         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1880                 neExceptions.NeutronException, ConnectionError) as e:
1881             if new_sf:
1882                 try:
1883                     self.neutron.delete_sfc_port_pair_group(
1884                         new_sf['port_pair_group']['id'])
1885                 except Exception:
1886                     self.logger.error(
1887                         'Creation of Service Function failed, with '
1888                         'subsequent deletion failure as well.')
1889             self._format_exception(e)
1890
1891     def get_sf(self, sf_id):
1892         self.logger.debug("Getting Service Function %s from VIM", sf_id)
1893         filter_dict = {"id": sf_id}
1894         sf_list = self.get_sf_list(filter_dict)
1895         if len(sf_list) == 0:
1896             raise vimconn.vimconnNotFoundException(
1897                 "Service Function '{}' not found".format(sf_id))
1898         elif len(sf_list) > 1:
1899             raise vimconn.vimconnConflictException(
1900                 "Found more than one Service Function with this criteria")
1901         sf = sf_list[0]
1902         return sf
1903
1904     def get_sf_list(self, filter_dict={}):
1905         self.logger.debug("Getting Service Function from VIM filter: '%s'",
1906                           str(filter_dict))
1907         try:
1908             self._reload_connection()
1909             filter_dict_os = filter_dict.copy()
1910             if self.api_version3 and "tenant_id" in filter_dict_os:
1911                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1912             sf_dict = self.neutron.list_sfc_port_pair_groups(**filter_dict_os)
1913             sf_list = sf_dict["port_pair_groups"]
1914             self.__sf_os2mano(sf_list)
1915             return sf_list
1916         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1917                 neExceptions.NeutronException, ConnectionError) as e:
1918             self._format_exception(e)
1919
1920     def delete_sf(self, sf_id):
1921         self.logger.debug("Deleting Service Function '%s' from VIM", sf_id)
1922         try:
1923             self._reload_connection()
1924             self.neutron.delete_sfc_port_pair_group(sf_id)
1925             return sf_id
1926         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1927                 ksExceptions.ClientException, neExceptions.NeutronException,
1928                 ConnectionError) as e:
1929             self._format_exception(e)
1930
1931     def new_sfp(self, name, classifications, sfs, sfc_encap=True, spi=None):
1932         self.logger.debug("Adding a new Service Function Path to VIM, "
1933                           "named '%s'", name)
1934         try:
1935             new_sfp = None
1936             self._reload_connection()
1937             # In networking-sfc the MPLS encapsulation is legacy
1938             # should be used when no full SFC Encapsulation is intended
1939             sfc_encap = 'mpls'
1940             if sfc_encap:
1941                 correlation = 'nsh'
1942             sfp_dict = {'name': name,
1943                         'flow_classifiers': classifications,
1944                         'port_pair_groups': sfs,
1945                         'chain_parameters': {'correlation': correlation}}
1946             if spi:
1947                 sfp_dict['chain_id'] = spi
1948             new_sfp = self.neutron.create_sfc_port_chain({'port_chain': sfp_dict})
1949             return new_sfp["port_chain"]["id"]
1950         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1951                 neExceptions.NeutronException, ConnectionError) as e:
1952             if new_sfp:
1953                 try:
1954                     self.neutron.delete_sfc_port_chain(new_sfp['port_chain']['id'])
1955                 except Exception:
1956                     self.logger.error(
1957                         'Creation of Service Function Path failed, with '
1958                         'subsequent deletion failure as well.')
1959             self._format_exception(e)
1960
1961     def get_sfp(self, sfp_id):
1962         self.logger.debug(" Getting Service Function Path %s from VIM", sfp_id)
1963         filter_dict = {"id": sfp_id}
1964         sfp_list = self.get_sfp_list(filter_dict)
1965         if len(sfp_list) == 0:
1966             raise vimconn.vimconnNotFoundException(
1967                 "Service Function Path '{}' not found".format(sfp_id))
1968         elif len(sfp_list) > 1:
1969             raise vimconn.vimconnConflictException(
1970                 "Found more than one Service Function Path with this criteria")
1971         sfp = sfp_list[0]
1972         return sfp
1973
1974     def get_sfp_list(self, filter_dict={}):
1975         self.logger.debug("Getting Service Function Paths from VIM filter: "
1976                           "'%s'", str(filter_dict))
1977         try:
1978             self._reload_connection()
1979             filter_dict_os = filter_dict.copy()
1980             if self.api_version3 and "tenant_id" in filter_dict_os:
1981                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1982             sfp_dict = self.neutron.list_sfc_port_chains(**filter_dict_os)
1983             sfp_list = sfp_dict["port_chains"]
1984             self.__sfp_os2mano(sfp_list)
1985             return sfp_list
1986         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1987                 neExceptions.NeutronException, ConnectionError) as e:
1988             self._format_exception(e)
1989
1990     def delete_sfp(self, sfp_id):
1991         self.logger.debug(
1992             "Deleting Service Function Path '%s' from VIM", sfp_id)
1993         try:
1994             self._reload_connection()
1995             self.neutron.delete_sfc_port_chain(sfp_id)
1996             return sfp_id
1997         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1998                 ksExceptions.ClientException, neExceptions.NeutronException,
1999                 ConnectionError) as e:
2000             self._format_exception(e)