osm_ro/vimconn_openstack.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 osconnector implements all the methods to interact with openstack using the python-neutronclient.
  26
  27 For the VNF forwarding graph, The OpenStack VIM connector calls the
  28 networking-sfc Neutron extension methods, whose resources are mapped
  29 to the VIM connector's SFC resources as follows:
  30 - Classification (OSM) -> Flow Classifier (Neutron)
  31 - Service Function Instance (OSM) -> Port Pair (Neutron)
  32 - Service Function (OSM) -> Port Pair Group (Neutron)
  33 - Service Function Path (OSM) -> Port Chain (Neutron)
  34 '''
  35 __author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes, xFlow Research, Igor D.C."
  36 __date__  = "$22-sep-2017 23:59:59$"
  37
  38 import vimconn
  39 # import json
  40 import logging
  41 import netaddr
  42 import time
  43 import yaml
  44 import random
  45 import re
  46 import copy
  47
  48 from novaclient import client as nClient, exceptions as nvExceptions
  49 from keystoneauth1.identity import v2, v3
  50 from keystoneauth1 import session
  51 import keystoneclient.exceptions as ksExceptions
  52 import keystoneclient.v3.client as ksClient_v3
  53 import keystoneclient.v2_0.client as ksClient_v2
  54 from glanceclient import client as glClient
  55 import glanceclient.client as gl1Client
  56 import glanceclient.exc as gl1Exceptions
  57 from  cinderclient import client as cClient
  58 from httplib import HTTPException
  59 from neutronclient.neutron import client as neClient
  60 from neutronclient.common import exceptions as neExceptions
  61 from requests.exceptions import ConnectionError
  62
  63
  64 """contain the openstack virtual machine status to openmano status"""
  65 vmStatus2manoFormat={'ACTIVE':'ACTIVE',
  66                      'PAUSED':'PAUSED',
  67                      'SUSPENDED': 'SUSPENDED',
  68                      'SHUTOFF':'INACTIVE',
  69                      'BUILD':'BUILD',
  70                      'ERROR':'ERROR','DELETED':'DELETED'
  71                      }
  72 netStatus2manoFormat={'ACTIVE':'ACTIVE','PAUSED':'PAUSED','INACTIVE':'INACTIVE','BUILD':'BUILD','ERROR':'ERROR','DELETED':'DELETED'
  73                      }
  74
  75 supportedClassificationTypes = ['legacy_flow_classifier']
  76
  77 #global var to have a timeout creating and deleting volumes
  78 volume_timeout = 600
  79 server_timeout = 600
  80
  81 class vimconnector(vimconn.vimconnector):
  82     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None,
  83                  log_level=None, config={}, persistent_info={}):
  84         '''using common constructor parameters. In this case
  85         'url' is the keystone authorization url,
  86         'url_admin' is not use
  87         '''
  88         api_version = config.get('APIversion')
  89         if api_version and api_version not in ('v3.3', 'v2.0', '2', '3'):
  90             raise vimconn.vimconnException("Invalid value '{}' for config:APIversion. "
  91                                            "Allowed values are 'v3.3', 'v2.0', '2' or '3'".format(api_version))
  92         vim_type = config.get('vim_type')
  93         if vim_type and vim_type not in ('vio', 'VIO'):
  94             raise vimconn.vimconnException("Invalid value '{}' for config:vim_type."
  95                             "Allowed values are 'vio' or 'VIO'".format(vim_type))
  96
  97         if config.get('dataplane_net_vlan_range') is not None:
  98             #validate vlan ranges provided by user
  99             self._validate_vlan_ranges(config.get('dataplane_net_vlan_range'))
 100
 101         vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level,
 102                                       config)
 103
 104         if self.config.get("insecure") and self.config.get("ca_cert"):
 105             raise vimconn.vimconnException("options insecure and ca_cert are mutually exclusive")
 106         self.verify = True
 107         if self.config.get("insecure"):
 108             self.verify = False
 109         if self.config.get("ca_cert"):
 110             self.verify = self.config.get("ca_cert")
 111
 112         if not url:
 113             raise TypeError('url param can not be NoneType')
 114         self.persistent_info = persistent_info
 115         self.availability_zone = persistent_info.get('availability_zone', None)
 116         self.session = persistent_info.get('session', {'reload_client': True})
 117         self.nova = self.session.get('nova')
 118         self.neutron = self.session.get('neutron')
 119         self.cinder = self.session.get('cinder')
 120         self.glance = self.session.get('glance')
 121         self.glancev1 = self.session.get('glancev1')
 122         self.keystone = self.session.get('keystone')
 123         self.api_version3 = self.session.get('api_version3')
 124         self.vim_type = self.config.get("vim_type")
 125         if self.vim_type:
 126             self.vim_type = self.vim_type.upper()
 127         if self.config.get("use_internal_endpoint"):
 128             self.endpoint_type = "internalURL"
 129         else:
 130             self.endpoint_type = None
 131
 132         self.logger = logging.getLogger('openmano.vim.openstack')
 133
 134         ####### VIO Specific Changes #########
 135         if self.vim_type == "VIO":
 136             self.logger = logging.getLogger('openmano.vim.vio')
 137
 138         if log_level:
 139             self.logger.setLevel( getattr(logging, log_level))
 140
 141     def __getitem__(self, index):
 142         """Get individuals parameters.
 143         Throw KeyError"""
 144         if index == 'project_domain_id':
 145             return self.config.get("project_domain_id")
 146         elif index == 'user_domain_id':
 147             return self.config.get("user_domain_id")
 148         else:
 149             return vimconn.vimconnector.__getitem__(self, index)
 150
 151     def __setitem__(self, index, value):
 152         """Set individuals parameters and it is marked as dirty so to force connection reload.
 153         Throw KeyError"""
 154         if index == 'project_domain_id':
 155             self.config["project_domain_id"] = value
 156         elif index == 'user_domain_id':
 157                 self.config["user_domain_id"] = value
 158         else:
 159             vimconn.vimconnector.__setitem__(self, index, value)
 160         self.session['reload_client'] = True
 161
 162     def _reload_connection(self):
 163         '''Called before any operation, it check if credentials has changed
 164         Throw keystoneclient.apiclient.exceptions.AuthorizationFailure
 165         '''
 166         #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-)
 167         if self.session['reload_client']:
 168             if self.config.get('APIversion'):
 169                 self.api_version3 = self.config['APIversion'] == 'v3.3' or self.config['APIversion'] == '3'
 170             else:  # get from ending auth_url that end with v3 or with v2.0
 171                 self.api_version3 =  self.url.endswith("/v3") or self.url.endswith("/v3/")
 172             self.session['api_version3'] = self.api_version3
 173             if self.api_version3:
 174                 if self.config.get('project_domain_id') or self.config.get('project_domain_name'):
 175                     project_domain_id_default = None
 176                 else:
 177                     project_domain_id_default = 'default'
 178                 if self.config.get('user_domain_id') or self.config.get('user_domain_name'):
 179                     user_domain_id_default = None
 180                 else:
 181                     user_domain_id_default = 'default'
 182                 auth = v3.Password(auth_url=self.url,
 183                                    username=self.user,
 184                                    password=self.passwd,
 185                                    project_name=self.tenant_name,
 186                                    project_id=self.tenant_id,
 187                                    project_domain_id=self.config.get('project_domain_id', project_domain_id_default),
 188                                    user_domain_id=self.config.get('user_domain_id', user_domain_id_default),
 189                                    project_domain_name=self.config.get('project_domain_name'),
 190                                    user_domain_name=self.config.get('user_domain_name'))
 191             else:
 192                 auth = v2.Password(auth_url=self.url,
 193                                    username=self.user,
 194                                    password=self.passwd,
 195                                    tenant_name=self.tenant_name,
 196                                    tenant_id=self.tenant_id)
 197             sess = session.Session(auth=auth, verify=self.verify)
 198             if self.api_version3:
 199                 self.keystone = ksClient_v3.Client(session=sess, endpoint_type=self.endpoint_type)
 200             else:
 201                 self.keystone = ksClient_v2.Client(session=sess, endpoint_type=self.endpoint_type)
 202             self.session['keystone'] = self.keystone
 203             # In order to enable microversion functionality an explicit microversion must be specified in 'config'.
 204             # This implementation approach is due to the warning message in
 205             # https://developer.openstack.org/api-guide/compute/microversions.html
 206             # where it is stated that microversion backwards compatibility is not guaranteed and clients should
 207             # always require an specific microversion.
 208             # To be able to use 'device role tagging' functionality define 'microversion: 2.32' in datacenter config
 209             version = self.config.get("microversion")
 210             if not version:
 211                 version = "2.1"
 212             self.nova = self.session['nova'] = nClient.Client(str(version), session=sess, endpoint_type=self.endpoint_type)
 213             self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess, endpoint_type=self.endpoint_type)
 214             self.cinder = self.session['cinder'] = cClient.Client(2, session=sess, endpoint_type=self.endpoint_type)
 215             if self.endpoint_type == "internalURL":
 216                 glance_service_id = self.keystone.services.list(name="glance")[0].id
 217                 glance_endpoint = self.keystone.endpoints.list(glance_service_id, interface="internal")[0].url
 218             else:
 219                 glance_endpoint = None
 220             self.glance = self.session['glance'] = glClient.Client(2, session=sess, endpoint=glance_endpoint)
 221             #using version 1 of glance client in new_image()
 222             self.glancev1 = self.session['glancev1'] = glClient.Client('1', session=sess,
 223                                                                        endpoint=glance_endpoint)
 224             self.session['reload_client'] = False
 225             self.persistent_info['session'] = self.session
 226             # add availablity zone info inside  self.persistent_info
 227             self._set_availablity_zones()
 228             self.persistent_info['availability_zone'] = self.availability_zone
 229
 230     def __net_os2mano(self, net_list_dict):
 231         '''Transform the net openstack format to mano format
 232         net_list_dict can be a list of dict or a single dict'''
 233         if type(net_list_dict) is dict:
 234             net_list_=(net_list_dict,)
 235         elif type(net_list_dict) is list:
 236             net_list_=net_list_dict
 237         else:
 238             raise TypeError("param net_list_dict must be a list or a dictionary")
 239         for net in net_list_:
 240             if net.get('provider:network_type') == "vlan":
 241                 net['type']='data'
 242             else:
 243                 net['type']='bridge'
 244
 245     def __classification_os2mano(self, class_list_dict):
 246         """Transform the openstack format (Flow Classifier) to mano format
 247         (Classification) class_list_dict can be a list of dict or a single dict
 248         """
 249         if isinstance(class_list_dict, dict):
 250             class_list_ = [class_list_dict]
 251         elif isinstance(class_list_dict, list):
 252             class_list_ = class_list_dict
 253         else:
 254             raise TypeError(
 255                 "param class_list_dict must be a list or a dictionary")
 256         for classification in class_list_:
 257             id = classification.pop('id')
 258             name = classification.pop('name')
 259             description = classification.pop('description')
 260             project_id = classification.pop('project_id')
 261             tenant_id = classification.pop('tenant_id')
 262             original_classification = copy.deepcopy(classification)
 263             classification.clear()
 264             classification['ctype'] = 'legacy_flow_classifier'
 265             classification['definition'] = original_classification
 266             classification['id'] = id
 267             classification['name'] = name
 268             classification['description'] = description
 269             classification['project_id'] = project_id
 270             classification['tenant_id'] = tenant_id
 271
 272     def __sfi_os2mano(self, sfi_list_dict):
 273         """Transform the openstack format (Port Pair) to mano format (SFI)
 274         sfi_list_dict can be a list of dict or a single dict
 275         """
 276         if isinstance(sfi_list_dict, dict):
 277             sfi_list_ = [sfi_list_dict]
 278         elif isinstance(sfi_list_dict, list):
 279             sfi_list_ = sfi_list_dict
 280         else:
 281             raise TypeError(
 282                 "param sfi_list_dict must be a list or a dictionary")
 283         for sfi in sfi_list_:
 284             sfi['ingress_ports'] = []
 285             sfi['egress_ports'] = []
 286             if sfi.get('ingress'):
 287                 sfi['ingress_ports'].append(sfi['ingress'])
 288             if sfi.get('egress'):
 289                 sfi['egress_ports'].append(sfi['egress'])
 290             del sfi['ingress']
 291             del sfi['egress']
 292             params = sfi.get('service_function_parameters')
 293             sfc_encap = False
 294             if params:
 295                 correlation = params.get('correlation')
 296                 if correlation:
 297                     sfc_encap = True
 298             sfi['sfc_encap'] = sfc_encap
 299             del sfi['service_function_parameters']
 300
 301     def __sf_os2mano(self, sf_list_dict):
 302         """Transform the openstack format (Port Pair Group) to mano format (SF)
 303         sf_list_dict can be a list of dict or a single dict
 304         """
 305         if isinstance(sf_list_dict, dict):
 306             sf_list_ = [sf_list_dict]
 307         elif isinstance(sf_list_dict, list):
 308             sf_list_ = sf_list_dict
 309         else:
 310             raise TypeError(
 311                 "param sf_list_dict must be a list or a dictionary")
 312         for sf in sf_list_:
 313             del sf['port_pair_group_parameters']
 314             sf['sfis'] = sf['port_pairs']
 315             del sf['port_pairs']
 316
 317     def __sfp_os2mano(self, sfp_list_dict):
 318         """Transform the openstack format (Port Chain) to mano format (SFP)
 319         sfp_list_dict can be a list of dict or a single dict
 320         """
 321         if isinstance(sfp_list_dict, dict):
 322             sfp_list_ = [sfp_list_dict]
 323         elif isinstance(sfp_list_dict, list):
 324             sfp_list_ = sfp_list_dict
 325         else:
 326             raise TypeError(
 327                 "param sfp_list_dict must be a list or a dictionary")
 328         for sfp in sfp_list_:
 329             params = sfp.pop('chain_parameters')
 330             sfc_encap = False
 331             if params:
 332                 correlation = params.get('correlation')
 333                 if correlation:
 334                     sfc_encap = True
 335             sfp['sfc_encap'] = sfc_encap
 336             sfp['spi'] = sfp.pop('chain_id')
 337             sfp['classifications'] = sfp.pop('flow_classifiers')
 338             sfp['service_functions'] = sfp.pop('port_pair_groups')
 339
 340     # placeholder for now; read TODO note below
 341     def _validate_classification(self, type, definition):
 342         # only legacy_flow_classifier Type is supported at this point
 343         return True
 344         # TODO(igordcard): this method should be an abstract method of an
 345         # abstract Classification class to be implemented by the specific
 346         # Types. Also, abstract vimconnector should call the validation
 347         # method before the implemented VIM connectors are called.
 348
 349     def _format_exception(self, exception):
 350         '''Transform a keystone, nova, neutron  exception into a vimconn exception'''
 351         if isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError,
 352                                   ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed
 353                                   )):
 354             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 355         elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException,
 356                                     neExceptions.NeutronException, nvExceptions.BadRequest)):
 357             raise vimconn.vimconnUnexpectedResponse(type(exception).__name__ + ": " + str(exception))
 358         elif isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound)):
 359             raise vimconn.vimconnNotFoundException(type(exception).__name__ + ": " + str(exception))
 360         elif isinstance(exception, nvExceptions.Conflict):
 361             raise vimconn.vimconnConflictException(type(exception).__name__ + ": " + str(exception))
 362         elif isinstance(exception, vimconn.vimconnException):
 363             raise exception
 364         else:  # ()
 365             self.logger.error("General Exception " + str(exception), exc_info=True)
 366             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 367
 368     def get_tenant_list(self, filter_dict={}):
 369         '''Obtain tenants of VIM
 370         filter_dict can contain the following keys:
 371             name: filter by tenant name
 372             id: filter by tenant uuid/id
 373             <other VIM specific>
 374         Returns the tenant list of dictionaries: [{'name':'<name>, 'id':'<id>, ...}, ...]
 375         '''
 376         self.logger.debug("Getting tenants from VIM filter: '%s'", str(filter_dict))
 377         try:
 378             self._reload_connection()
 379             if self.api_version3:
 380                 project_class_list = self.keystone.projects.list(name=filter_dict.get("name"))
 381             else:
 382                 project_class_list = self.keystone.tenants.findall(**filter_dict)
 383             project_list=[]
 384             for project in project_class_list:
 385                 if filter_dict.get('id') and filter_dict["id"] != project.id:
 386                     continue
 387                 project_list.append(project.to_dict())
 388             return project_list
 389         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError) as e:
 390             self._format_exception(e)
 391
 392     def new_tenant(self, tenant_name, tenant_description):
 393         '''Adds a new tenant to openstack VIM. Returns the tenant identifier'''
 394         self.logger.debug("Adding a new tenant name: %s", tenant_name)
 395         try:
 396             self._reload_connection()
 397             if self.api_version3:
 398                 project = self.keystone.projects.create(tenant_name, self.config.get("project_domain_id", "default"),
 399                                                         description=tenant_description, is_domain=False)
 400             else:
 401                 project = self.keystone.tenants.create(tenant_name, tenant_description)
 402             return project.id
 403         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 404             self._format_exception(e)
 405
 406     def delete_tenant(self, tenant_id):
 407         '''Delete a tenant from openstack VIM. Returns the old tenant identifier'''
 408         self.logger.debug("Deleting tenant %s from VIM", tenant_id)
 409         try:
 410             self._reload_connection()
 411             if self.api_version3:
 412                 self.keystone.projects.delete(tenant_id)
 413             else:
 414                 self.keystone.tenants.delete(tenant_id)
 415             return tenant_id
 416         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 417             self._format_exception(e)
 418
 419     def new_network(self,net_name, net_type, ip_profile=None, shared=False, vlan=None):
 420         '''Adds a tenant network to VIM. Returns the network identifier'''
 421         self.logger.debug("Adding a new network to VIM name '%s', type '%s'", net_name, net_type)
 422         #self.logger.debug(">>>>>>>>>>>>>>>>>> IP profile %s", str(ip_profile))
 423         try:
 424             new_net = None
 425             self._reload_connection()
 426             network_dict = {'name': net_name, 'admin_state_up': True}
 427             if net_type=="data" or net_type=="ptp":
 428                 if self.config.get('dataplane_physical_net') == None:
 429                     raise vimconn.vimconnConflictException("You must provide a 'dataplane_physical_net' at config value before creating sriov network")
 430                 network_dict["provider:physical_network"] = self.config['dataplane_physical_net'] #"physnet_sriov" #TODO physical
 431                 network_dict["provider:network_type"]     = "vlan"
 432                 if vlan!=None:
 433                     network_dict["provider:network_type"] = vlan
 434
 435                 ####### VIO Specific Changes #########
 436                 if self.vim_type == "VIO":
 437                     if vlan is not None:
 438                         network_dict["provider:segmentation_id"] = vlan
 439                     else:
 440                         if self.config.get('dataplane_net_vlan_range') is None:
 441                             raise vimconn.vimconnConflictException("You must provide "\
 442                                 "'dataplane_net_vlan_range' in format [start_ID - end_ID]"\
 443                                 "at config value before creating sriov network with vlan tag")
 444
 445                         network_dict["provider:segmentation_id"] = self._genrate_vlanID()
 446
 447             network_dict["shared"]=shared
 448             new_net=self.neutron.create_network({'network':network_dict})
 449             #print new_net
 450             #create subnetwork, even if there is no profile
 451             if not ip_profile:
 452                 ip_profile = {}
 453             if not ip_profile.get('subnet_address'):
 454                 #Fake subnet is required
 455                 subnet_rand = random.randint(0, 255)
 456                 ip_profile['subnet_address'] = "192.168.{}.0/24".format(subnet_rand)
 457             if 'ip_version' not in ip_profile:
 458                 ip_profile['ip_version'] = "IPv4"
 459             subnet = {"name":net_name+"-subnet",
 460                     "network_id": new_net["network"]["id"],
 461                     "ip_version": 4 if ip_profile['ip_version']=="IPv4" else 6,
 462                     "cidr": ip_profile['subnet_address']
 463                     }
 464             # Gateway should be set to None if not needed. Otherwise openstack assigns one by default
 465             if ip_profile.get('gateway_address'):
 466                 subnet['gateway_ip'] = ip_profile.get('gateway_address')
 467             if ip_profile.get('dns_address'):
 468                 subnet['dns_nameservers'] = ip_profile['dns_address'].split(";")
 469             if 'dhcp_enabled' in ip_profile:
 470                 subnet['enable_dhcp'] = False if \
 471                     ip_profile['dhcp_enabled']=="false" or ip_profile['dhcp_enabled']==False else True
 472             if ip_profile.get('dhcp_start_address'):
 473                 subnet['allocation_pools'] = []
 474                 subnet['allocation_pools'].append(dict())
 475                 subnet['allocation_pools'][0]['start'] = ip_profile['dhcp_start_address']
 476             if ip_profile.get('dhcp_count'):
 477                 #parts = ip_profile['dhcp_start_address'].split('.')
 478                 #ip_int = (int(parts[0]) << 24) + (int(parts[1]) << 16) + (int(parts[2]) << 8) + int(parts[3])
 479                 ip_int = int(netaddr.IPAddress(ip_profile['dhcp_start_address']))
 480                 ip_int += ip_profile['dhcp_count'] - 1
 481                 ip_str = str(netaddr.IPAddress(ip_int))
 482                 subnet['allocation_pools'][0]['end'] = ip_str
 483             #self.logger.debug(">>>>>>>>>>>>>>>>>> Subnet: %s", str(subnet))
 484             self.neutron.create_subnet({"subnet": subnet} )
 485             return new_net["network"]["id"]
 486         except Exception as e:
 487             if new_net:
 488                 self.neutron.delete_network(new_net['network']['id'])
 489             self._format_exception(e)
 490
 491     def get_network_list(self, filter_dict={}):
 492         '''Obtain tenant networks of VIM
 493         Filter_dict can be:
 494             name: network name
 495             id: network uuid
 496             shared: boolean
 497             tenant_id: tenant
 498             admin_state_up: boolean
 499             status: 'ACTIVE'
 500         Returns the network list of dictionaries
 501         '''
 502         self.logger.debug("Getting network from VIM filter: '%s'", str(filter_dict))
 503         try:
 504             self._reload_connection()
 505             filter_dict_os = filter_dict.copy()
 506             if self.api_version3 and "tenant_id" in filter_dict_os:
 507                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')  #T ODO check
 508             net_dict = self.neutron.list_networks(**filter_dict_os)
 509             net_list = net_dict["networks"]
 510             self.__net_os2mano(net_list)
 511             return net_list
 512         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 513             self._format_exception(e)
 514
 515     def get_network(self, net_id):
 516         '''Obtain details of network from VIM
 517         Returns the network information from a network id'''
 518         self.logger.debug(" Getting tenant network %s from VIM", net_id)
 519         filter_dict={"id": net_id}
 520         net_list = self.get_network_list(filter_dict)
 521         if len(net_list)==0:
 522             raise vimconn.vimconnNotFoundException("Network '{}' not found".format(net_id))
 523         elif len(net_list)>1:
 524             raise vimconn.vimconnConflictException("Found more than one network with this criteria")
 525         net = net_list[0]
 526         subnets=[]
 527         for subnet_id in net.get("subnets", () ):
 528             try:
 529                 subnet = self.neutron.show_subnet(subnet_id)
 530             except Exception as e:
 531                 self.logger.error("osconnector.get_network(): Error getting subnet %s %s" % (net_id, str(e)))
 532                 subnet = {"id": subnet_id, "fault": str(e)}
 533             subnets.append(subnet)
 534         net["subnets"] = subnets
 535         net["encapsulation"] = net.get('provider:network_type')
 536         net["segmentation_id"] = net.get('provider:segmentation_id')
 537         return net
 538
 539     def delete_network(self, net_id):
 540         '''Deletes a tenant network from VIM. Returns the old network identifier'''
 541         self.logger.debug("Deleting network '%s' from VIM", net_id)
 542         try:
 543             self._reload_connection()
 544             #delete VM ports attached to this networks before the network
 545             ports = self.neutron.list_ports(network_id=net_id)
 546             for p in ports['ports']:
 547                 try:
 548                     self.neutron.delete_port(p["id"])
 549                 except Exception as e:
 550                     self.logger.error("Error deleting port %s: %s", p["id"], str(e))
 551             self.neutron.delete_network(net_id)
 552             return net_id
 553         except (neExceptions.ConnectionFailed, neExceptions.NetworkNotFoundClient, neExceptions.NeutronException,
 554                 ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 555             self._format_exception(e)
 556
 557     def refresh_nets_status(self, net_list):
 558         '''Get the status of the networks
 559            Params: the list of network identifiers
 560            Returns a dictionary with:
 561                 net_id:         #VIM id of this network
 562                     status:     #Mandatory. Text with one of:
 563                                 #  DELETED (not found at vim)
 564                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 565                                 #  OTHER (Vim reported other status not understood)
 566                                 #  ERROR (VIM indicates an ERROR status)
 567                                 #  ACTIVE, INACTIVE, DOWN (admin down),
 568                                 #  BUILD (on building process)
 569                                 #
 570                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 571                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 572
 573         '''
 574         net_dict={}
 575         for net_id in net_list:
 576             net = {}
 577             try:
 578                 net_vim = self.get_network(net_id)
 579                 if net_vim['status'] in netStatus2manoFormat:
 580                     net["status"] = netStatus2manoFormat[ net_vim['status'] ]
 581                 else:
 582                     net["status"] = "OTHER"
 583                     net["error_msg"] = "VIM status reported " + net_vim['status']
 584
 585                 if net['status'] == "ACTIVE" and not net_vim['admin_state_up']:
 586                     net['status'] = 'DOWN'
 587                 try:
 588                     net['vim_info'] = yaml.safe_dump(net_vim, default_flow_style=True, width=256)
 589                 except yaml.representer.RepresenterError:
 590                     net['vim_info'] = str(net_vim)
 591                 if net_vim.get('fault'):  #TODO
 592                     net['error_msg'] = str(net_vim['fault'])
 593             except vimconn.vimconnNotFoundException as e:
 594                 self.logger.error("Exception getting net status: %s", str(e))
 595                 net['status'] = "DELETED"
 596                 net['error_msg'] = str(e)
 597             except vimconn.vimconnException as e:
 598                 self.logger.error("Exception getting net status: %s", str(e))
 599                 net['status'] = "VIM_ERROR"
 600                 net['error_msg'] = str(e)
 601             net_dict[net_id] = net
 602         return net_dict
 603
 604     def get_flavor(self, flavor_id):
 605         '''Obtain flavor details from the  VIM. Returns the flavor dict details'''
 606         self.logger.debug("Getting flavor '%s'", flavor_id)
 607         try:
 608             self._reload_connection()
 609             flavor = self.nova.flavors.find(id=flavor_id)
 610             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 611             return flavor.to_dict()
 612         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 613             self._format_exception(e)
 614
 615     def get_flavor_id_from_data(self, flavor_dict):
 616         """Obtain flavor id that match the flavor description
 617            Returns the flavor_id or raises a vimconnNotFoundException
 618            flavor_dict: contains the required ram, vcpus, disk
 619            If 'use_existing_flavors' is set to True at config, the closer flavor that provides same or more ram, vcpus
 620                 and disk is returned. Otherwise a flavor with exactly same ram, vcpus and disk is returned or a
 621                 vimconnNotFoundException is raised
 622         """
 623         exact_match = False if self.config.get('use_existing_flavors') else True
 624         try:
 625             self._reload_connection()
 626             flavor_candidate_id = None
 627             flavor_candidate_data = (10000, 10000, 10000)
 628             flavor_target = (flavor_dict["ram"], flavor_dict["vcpus"], flavor_dict["disk"])
 629             # numa=None
 630             numas = flavor_dict.get("extended", {}).get("numas")
 631             if numas:
 632                 #TODO
 633                 raise vimconn.vimconnNotFoundException("Flavor with EPA still not implemted")
 634                 # if len(numas) > 1:
 635                 #     raise vimconn.vimconnNotFoundException("Cannot find any flavor with more than one numa")
 636                 # numa=numas[0]
 637                 # numas = extended.get("numas")
 638             for flavor in self.nova.flavors.list():
 639                 epa = flavor.get_keys()
 640                 if epa:
 641                     continue
 642                     # TODO
 643                 flavor_data = (flavor.ram, flavor.vcpus, flavor.disk)
 644                 if flavor_data == flavor_target:
 645                     return flavor.id
 646                 elif not exact_match and flavor_target < flavor_data < flavor_candidate_data:
 647                     flavor_candidate_id = flavor.id
 648                     flavor_candidate_data = flavor_data
 649             if not exact_match and flavor_candidate_id:
 650                 return flavor_candidate_id
 651             raise vimconn.vimconnNotFoundException("Cannot find any flavor matching '{}'".format(str(flavor_dict)))
 652         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 653             self._format_exception(e)
 654
 655     def new_flavor(self, flavor_data, change_name_if_used=True):
 656         '''Adds a tenant flavor to openstack VIM
 657         if change_name_if_used is True, it will change name in case of conflict, because it is not supported name repetition
 658         Returns the flavor identifier
 659         '''
 660         self.logger.debug("Adding flavor '%s'", str(flavor_data))
 661         retry=0
 662         max_retries=3
 663         name_suffix = 0
 664         name=flavor_data['name']
 665         while retry<max_retries:
 666             retry+=1
 667             try:
 668                 self._reload_connection()
 669                 if change_name_if_used:
 670                     #get used names
 671                     fl_names=[]
 672                     fl=self.nova.flavors.list()
 673                     for f in fl:
 674                         fl_names.append(f.name)
 675                     while name in fl_names:
 676                         name_suffix += 1
 677                         name = flavor_data['name']+"-" + str(name_suffix)
 678
 679                 ram = flavor_data.get('ram',64)
 680                 vcpus = flavor_data.get('vcpus',1)
 681                 numa_properties=None
 682
 683                 extended = flavor_data.get("extended")
 684                 if extended:
 685                     numas=extended.get("numas")
 686                     if numas:
 687                         numa_nodes = len(numas)
 688                         if numa_nodes > 1:
 689                             return -1, "Can not add flavor with more than one numa"
 690                         numa_properties = {"hw:numa_nodes":str(numa_nodes)}
 691                         numa_properties["hw:mem_page_size"] = "large"
 692                         numa_properties["hw:cpu_policy"] = "dedicated"
 693                         numa_properties["hw:numa_mempolicy"] = "strict"
 694                         if self.vim_type == "VIO":
 695                             numa_properties["vmware:extra_config"] = '{"numa.nodeAffinity":"0"}'
 696                             numa_properties["vmware:latency_sensitivity_level"] = "high"
 697                         for numa in numas:
 698                             #overwrite ram and vcpus
 699                             #check if key 'memory' is present in numa else use ram value at flavor
 700                             if 'memory' in numa:
 701                                 ram = numa['memory']*1024
 702                             #See for reference: https://specs.openstack.org/openstack/nova-specs/specs/mitaka/implemented/virt-driver-cpu-thread-pinning.html
 703                             if 'paired-threads' in numa:
 704                                 vcpus = numa['paired-threads']*2
 705                                 #cpu_thread_policy "require" implies that the compute node must have an STM architecture
 706                                 numa_properties["hw:cpu_thread_policy"] = "require"
 707                                 numa_properties["hw:cpu_policy"] = "dedicated"
 708                             elif 'cores' in numa:
 709                                 vcpus = numa['cores']
 710                                 # cpu_thread_policy "prefer" implies that the host must not have an SMT architecture, or a non-SMT architecture will be emulated
 711                                 numa_properties["hw:cpu_thread_policy"] = "isolate"
 712                                 numa_properties["hw:cpu_policy"] = "dedicated"
 713                             elif 'threads' in numa:
 714                                 vcpus = numa['threads']
 715                                 # cpu_thread_policy "prefer" implies that the host may or may not have an SMT architecture
 716                                 numa_properties["hw:cpu_thread_policy"] = "prefer"
 717                                 numa_properties["hw:cpu_policy"] = "dedicated"
 718                             # for interface in numa.get("interfaces",() ):
 719                             #     if interface["dedicated"]=="yes":
 720                             #         raise vimconn.vimconnException("Passthrough interfaces are not supported for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable)
 721                             #     #TODO, add the key 'pci_passthrough:alias"="<label at config>:<number ifaces>"' when a way to connect it is available
 722
 723                 #create flavor
 724                 new_flavor=self.nova.flavors.create(name,
 725                                 ram,
 726                                 vcpus,
 727                                 flavor_data.get('disk',0),
 728                                 is_public=flavor_data.get('is_public', True)
 729                             )
 730                 #add metadata
 731                 if numa_properties:
 732                     new_flavor.set_keys(numa_properties)
 733                 return new_flavor.id
 734             except nvExceptions.Conflict as e:
 735                 if change_name_if_used and retry < max_retries:
 736                     continue
 737                 self._format_exception(e)
 738             #except nvExceptions.BadRequest as e:
 739             except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 740                 self._format_exception(e)
 741
 742     def delete_flavor(self,flavor_id):
 743         '''Deletes a tenant flavor from openstack VIM. Returns the old flavor_id
 744         '''
 745         try:
 746             self._reload_connection()
 747             self.nova.flavors.delete(flavor_id)
 748             return flavor_id
 749         #except nvExceptions.BadRequest as e:
 750         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 751             self._format_exception(e)
 752
 753     def new_image(self,image_dict):
 754         '''
 755         Adds a tenant image to VIM. imge_dict is a dictionary with:
 756             name: name
 757             disk_format: qcow2, vhd, vmdk, raw (by default), ...
 758             location: path or URI
 759             public: "yes" or "no"
 760             metadata: metadata of the image
 761         Returns the image_id
 762         '''
 763         retry=0
 764         max_retries=3
 765         while retry<max_retries:
 766             retry+=1
 767             try:
 768                 self._reload_connection()
 769                 #determine format  http://docs.openstack.org/developer/glance/formats.html
 770                 if "disk_format" in image_dict:
 771                     disk_format=image_dict["disk_format"]
 772                 else: #autodiscover based on extension
 773                     if image_dict['location'][-6:]==".qcow2":
 774                         disk_format="qcow2"
 775                     elif image_dict['location'][-4:]==".vhd":
 776                         disk_format="vhd"
 777                     elif image_dict['location'][-5:]==".vmdk":
 778                         disk_format="vmdk"
 779                     elif image_dict['location'][-4:]==".vdi":
 780                         disk_format="vdi"
 781                     elif image_dict['location'][-4:]==".iso":
 782                         disk_format="iso"
 783                     elif image_dict['location'][-4:]==".aki":
 784                         disk_format="aki"
 785                     elif image_dict['location'][-4:]==".ari":
 786                         disk_format="ari"
 787                     elif image_dict['location'][-4:]==".ami":
 788                         disk_format="ami"
 789                     else:
 790                         disk_format="raw"
 791                 self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location'])
 792                 if image_dict['location'][0:4]=="http":
 793                     new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 794                             container_format="bare", location=image_dict['location'], disk_format=disk_format)
 795                 else: #local path
 796                     with open(image_dict['location']) as fimage:
 797                         new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 798                             container_format="bare", data=fimage, disk_format=disk_format)
 799                 #insert metadata. We cannot use 'new_image.properties.setdefault'
 800                 #because nova and glance are "INDEPENDENT" and we are using nova for reading metadata
 801                 new_image_nova=self.nova.images.find(id=new_image.id)
 802                 new_image_nova.metadata.setdefault('location',image_dict['location'])
 803                 metadata_to_load = image_dict.get('metadata')
 804                 if metadata_to_load:
 805                     for k,v in yaml.load(metadata_to_load).iteritems():
 806                         new_image_nova.metadata.setdefault(k,v)
 807                 return new_image.id
 808             except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e:
 809                 self._format_exception(e)
 810             except (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 811                 if retry==max_retries:
 812                     continue
 813                 self._format_exception(e)
 814             except IOError as e:  #can not open the file
 815                 raise vimconn.vimconnConnectionException(type(e).__name__ + ": " + str(e)+ " for " + image_dict['location'],
 816                                                          http_code=vimconn.HTTP_Bad_Request)
 817
 818     def delete_image(self, image_id):
 819         '''Deletes a tenant image from openstack VIM. Returns the old id
 820         '''
 821         try:
 822             self._reload_connection()
 823             self.nova.images.delete(image_id)
 824             return image_id
 825         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e: #TODO remove
 826             self._format_exception(e)
 827
 828     def get_image_id_from_path(self, path):
 829         '''Get the image id from image path in the VIM database. Returns the image_id'''
 830         try:
 831             self._reload_connection()
 832             images = self.nova.images.list()
 833             for image in images:
 834                 if image.metadata.get("location")==path:
 835                     return image.id
 836             raise vimconn.vimconnNotFoundException("image with location '{}' not found".format( path))
 837         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 838             self._format_exception(e)
 839
 840     def get_image_list(self, filter_dict={}):
 841         '''Obtain tenant images from VIM
 842         Filter_dict can be:
 843             id: image id
 844             name: image name
 845             checksum: image checksum
 846         Returns the image list of dictionaries:
 847             [{<the fields at Filter_dict plus some VIM specific>}, ...]
 848             List can be empty
 849         '''
 850         self.logger.debug("Getting image list from VIM filter: '%s'", str(filter_dict))
 851         try:
 852             self._reload_connection()
 853             filter_dict_os = filter_dict.copy()
 854             #First we filter by the available filter fields: name, id. The others are removed.
 855             filter_dict_os.pop('checksum', None)
 856             image_list = self.nova.images.findall(**filter_dict_os)
 857             if len(image_list) == 0:
 858                 return []
 859             #Then we filter by the rest of filter fields: checksum
 860             filtered_list = []
 861             for image in image_list:
 862                 try:
 863                     image_class = self.glance.images.get(image.id)
 864                     if 'checksum' not in filter_dict or image_class['checksum'] == filter_dict.get('checksum'):
 865                         filtered_list.append(image_class.copy())
 866                 except gl1Exceptions.HTTPNotFound:
 867                     pass
 868             return filtered_list
 869         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 870             self._format_exception(e)
 871
 872     def __wait_for_vm(self, vm_id, status):
 873         """wait until vm is in the desired status and return True.
 874         If the VM gets in ERROR status, return false.
 875         If the timeout is reached generate an exception"""
 876         elapsed_time = 0
 877         while elapsed_time < server_timeout:
 878             vm_status = self.nova.servers.get(vm_id).status
 879             if vm_status == status:
 880                 return True
 881             if vm_status == 'ERROR':
 882                 return False
 883             time.sleep(1)
 884             elapsed_time += 1
 885
 886         # if we exceeded the timeout rollback
 887         if elapsed_time >= server_timeout:
 888             raise vimconn.vimconnException('Timeout waiting for instance ' + vm_id + ' to get ' + status,
 889                                            http_code=vimconn.HTTP_Request_Timeout)
 890
 891     def _get_openstack_availablity_zones(self):
 892         """
 893         Get from openstack availability zones available
 894         :return:
 895         """
 896         try:
 897             openstack_availability_zone = self.nova.availability_zones.list()
 898             openstack_availability_zone = [str(zone.zoneName) for zone in openstack_availability_zone
 899                                            if zone.zoneName != 'internal']
 900             return openstack_availability_zone
 901         except Exception as e:
 902             return None
 903
 904     def _set_availablity_zones(self):
 905         """
 906         Set vim availablity zone
 907         :return:
 908         """
 909
 910         if 'availability_zone' in self.config:
 911             vim_availability_zones = self.config.get('availability_zone')
 912             if isinstance(vim_availability_zones, str):
 913                 self.availability_zone = [vim_availability_zones]
 914             elif isinstance(vim_availability_zones, list):
 915                 self.availability_zone = vim_availability_zones
 916         else:
 917             self.availability_zone = self._get_openstack_availablity_zones()
 918
 919     def _get_vm_availability_zone(self, availability_zone_index, availability_zone_list):
 920         """
 921         Return thge availability zone to be used by the created VM.
 922         :return: The VIM availability zone to be used or None
 923         """
 924         if availability_zone_index is None:
 925             if not self.config.get('availability_zone'):
 926                 return None
 927             elif isinstance(self.config.get('availability_zone'), str):
 928                 return self.config['availability_zone']
 929             else:
 930                 # TODO consider using a different parameter at config for default AV and AV list match
 931                 return self.config['availability_zone'][0]
 932
 933         vim_availability_zones = self.availability_zone
 934         # check if VIM offer enough availability zones describe in the VNFD
 935         if vim_availability_zones and len(availability_zone_list) <= len(vim_availability_zones):
 936             # check if all the names of NFV AV match VIM AV names
 937             match_by_index = False
 938             for av in availability_zone_list:
 939                 if av not in vim_availability_zones:
 940                     match_by_index = True
 941                     break
 942             if match_by_index:
 943                 return vim_availability_zones[availability_zone_index]
 944             else:
 945                 return availability_zone_list[availability_zone_index]
 946         else:
 947             raise vimconn.vimconnConflictException("No enough availability zones at VIM for this deployment")
 948
 949     def new_vminstance(self, name, description, start, image_id, flavor_id, net_list, cloud_config=None, disk_list=None,
 950                        availability_zone_index=None, availability_zone_list=None):
 951         """Adds a VM instance to VIM
 952         Params:
 953             start: indicates if VM must start or boot in pause mode. Ignored
 954             image_id,flavor_id: iamge and flavor uuid
 955             net_list: list of interfaces, each one is a dictionary with:
 956                 name:
 957                 net_id: network uuid to connect
 958                 vpci: virtual vcpi to assign, ignored because openstack lack #TODO
 959                 model: interface model, ignored #TODO
 960                 mac_address: used for  SR-IOV ifaces #TODO for other types
 961                 use: 'data', 'bridge',  'mgmt'
 962                 type: 'virtual', 'PCI-PASSTHROUGH'('PF'), 'SR-IOV'('VF'), 'VFnotShared'
 963                 vim_id: filled/added by this function
 964                 floating_ip: True/False (or it can be None)
 965             'cloud_config': (optional) dictionary with:
 966             'key-pairs': (optional) list of strings with the public key to be inserted to the default user
 967             'users': (optional) list of users to be inserted, each item is a dict with:
 968                 'name': (mandatory) user name,
 969                 'key-pairs': (optional) list of strings with the public key to be inserted to the user
 970             'user-data': (optional) string is a text script to be passed directly to cloud-init
 971             'config-files': (optional). List of files to be transferred. Each item is a dict with:
 972                 'dest': (mandatory) string with the destination absolute path
 973                 'encoding': (optional, by default text). Can be one of:
 974                     'b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64'
 975                 'content' (mandatory): string with the content of the file
 976                 'permissions': (optional) string with file permissions, typically octal notation '0644'
 977                 'owner': (optional) file owner, string with the format 'owner:group'
 978             'boot-data-drive': boolean to indicate if user-data must be passed using a boot drive (hard disk)
 979             'disk_list': (optional) list with additional disks to the VM. Each item is a dict with:
 980                 'image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted
 981                 'size': (mandatory) string with the size of the disk in GB
 982             availability_zone_index: Index of availability_zone_list to use for this this VM. None if not AV required
 983             availability_zone_list: list of availability zones given by user in the VNFD descriptor.  Ignore if
 984                 availability_zone_index is None
 985                 #TODO ip, security groups
 986         Returns a tuple with the instance identifier and created_items or raises an exception on error
 987             created_items can be None or a dictionary where this method can include key-values that will be passed to
 988             the method delete_vminstance and action_vminstance. Can be used to store created ports, volumes, etc.
 989             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
 990             as not present.
 991         """
 992         self.logger.debug("new_vminstance input: image='%s' flavor='%s' nics='%s'",image_id, flavor_id,str(net_list))
 993         try:
 994             server = None
 995             created_items = {}
 996             # metadata = {}
 997             net_list_vim = []
 998             external_network = []   # list of external networks to be connected to instance, later on used to create floating_ip
 999             no_secured_ports = []   # List of port-is with port-security disabled
1000             self._reload_connection()
1001             # metadata_vpci = {}   # For a specific neutron plugin
1002             block_device_mapping = None
1003             for net in net_list:
1004                 if not net.get("net_id"):   # skip non connected iface
1005                     continue
1006
1007                 port_dict={
1008                     "network_id": net["net_id"],
1009                     "name": net.get("name"),
1010                     "admin_state_up": True
1011                 }
1012                 if net["type"]=="virtual":
1013                     pass
1014                     # if "vpci" in net:
1015                     #     metadata_vpci[ net["net_id"] ] = [[ net["vpci"], "" ]]
1016                 elif net["type"] == "VF" or net["type"] == "SR-IOV":  # for VF
1017                     # if "vpci" in net:
1018                     #     if "VF" not in metadata_vpci:
1019                     #         metadata_vpci["VF"]=[]
1020                     #     metadata_vpci["VF"].append([ net["vpci"], "" ])
1021                     port_dict["binding:vnic_type"]="direct"
1022                     # VIO specific Changes
1023                     if self.vim_type == "VIO":
1024                         # Need to create port with port_security_enabled = False and no-security-groups
1025                         port_dict["port_security_enabled"]=False
1026                         port_dict["provider_security_groups"]=[]
1027                         port_dict["security_groups"]=[]
1028                 else:   # For PT PCI-PASSTHROUGH
1029                     # VIO specific Changes
1030                     # Current VIO release does not support port with type 'direct-physical'
1031                     # So no need to create virtual port in case of PCI-device.
1032                     # Will update port_dict code when support gets added in next VIO release
1033                     if self.vim_type == "VIO":
1034                         raise vimconn.vimconnNotSupportedException(
1035                             "Current VIO release does not support full passthrough (PT)")
1036                     # if "vpci" in net:
1037                     #     if "PF" not in metadata_vpci:
1038                     #         metadata_vpci["PF"]=[]
1039                     #     metadata_vpci["PF"].append([ net["vpci"], "" ])
1040                     port_dict["binding:vnic_type"]="direct-physical"
1041                 if not port_dict["name"]:
1042                     port_dict["name"]=name
1043                 if net.get("mac_address"):
1044                     port_dict["mac_address"]=net["mac_address"]
1045                 if net.get("ip_address"):
1046                     port_dict["fixed_ips"] = [{'ip_address': net["ip_address"]}]
1047                     # TODO add 'subnet_id': <subnet_id>
1048                 new_port = self.neutron.create_port({"port": port_dict })
1049                 created_items["port:" + str(new_port["port"]["id"])] = True
1050                 net["mac_adress"] = new_port["port"]["mac_address"]
1051                 net["vim_id"] = new_port["port"]["id"]
1052                 # if try to use a network without subnetwork, it will return a emtpy list
1053                 fixed_ips = new_port["port"].get("fixed_ips")
1054                 if fixed_ips:
1055                     net["ip"] = fixed_ips[0].get("ip_address")
1056                 else:
1057                     net["ip"] = None
1058
1059                 port = {"port-id": new_port["port"]["id"]}
1060                 if float(self.nova.api_version.get_string()) >= 2.32:
1061                     port["tag"] = new_port["port"]["name"]
1062                 net_list_vim.append(port)
1063
1064                 if net.get('floating_ip', False):
1065                     net['exit_on_floating_ip_error'] = True
1066                     external_network.append(net)
1067                 elif net['use'] == 'mgmt' and self.config.get('use_floating_ip'):
1068                     net['exit_on_floating_ip_error'] = False
1069                     external_network.append(net)
1070                     net['floating_ip'] = self.config.get('use_floating_ip')
1071
1072                 # If port security is disabled when the port has not yet been attached to the VM, then all vm traffic is dropped.
1073                 # As a workaround we wait until the VM is active and then disable the port-security
1074                 if net.get("port_security") == False and not self.config.get("no_port_security_extension"):
1075                     no_secured_ports.append(new_port["port"]["id"])
1076
1077             # if metadata_vpci:
1078             #     metadata = {"pci_assignement": json.dumps(metadata_vpci)}
1079             #     if len(metadata["pci_assignement"]) >255:
1080             #         #limit the metadata size
1081             #         #metadata["pci_assignement"] = metadata["pci_assignement"][0:255]
1082             #         self.logger.warn("Metadata deleted since it exceeds the expected length (255) ")
1083             #         metadata = {}
1084
1085             self.logger.debug("name '%s' image_id '%s'flavor_id '%s' net_list_vim '%s' description '%s'",
1086                               name, image_id, flavor_id, str(net_list_vim), description)
1087
1088             security_groups = self.config.get('security_groups')
1089             if type(security_groups) is str:
1090                 security_groups = ( security_groups, )
1091             # cloud config
1092             config_drive, userdata = self._create_user_data(cloud_config)
1093
1094             # Create additional volumes in case these are present in disk_list
1095             base_disk_index = ord('b')
1096             if disk_list != None:
1097                 block_device_mapping = {}
1098                 for disk in disk_list:
1099                     if 'image_id' in disk:
1100                         volume = self.cinder.volumes.create(size = disk['size'],name = name + '_vd' +
1101                                     chr(base_disk_index), imageRef = disk['image_id'])
1102                     else:
1103                         volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1104                                     chr(base_disk_index))
1105                     created_items["volume:" + str(volume.id)] = True
1106                     block_device_mapping['_vd' +  chr(base_disk_index)] = volume.id
1107                     base_disk_index += 1
1108
1109                 # Wait until volumes are with status available
1110                 keep_waiting = True
1111                 elapsed_time = 0
1112                 while keep_waiting and elapsed_time < volume_timeout:
1113                     keep_waiting = False
1114                     for volume_id in block_device_mapping.itervalues():
1115                         if self.cinder.volumes.get(volume_id).status != 'available':
1116                             keep_waiting = True
1117                     if keep_waiting:
1118                         time.sleep(1)
1119                         elapsed_time += 1
1120
1121                 # If we exceeded the timeout rollback
1122                 if elapsed_time >= volume_timeout:
1123                     raise vimconn.vimconnException('Timeout creating volumes for instance ' + name,
1124                                                    http_code=vimconn.HTTP_Request_Timeout)
1125             # get availability Zone
1126             vm_av_zone = self._get_vm_availability_zone(availability_zone_index, availability_zone_list)
1127
1128             self.logger.debug("nova.servers.create({}, {}, {}, nics={}, security_groups={}, "
1129                               "availability_zone={}, key_name={}, userdata={}, config_drive={}, "
1130                               "block_device_mapping={})".format(name, image_id, flavor_id, net_list_vim,
1131                                                                 security_groups, vm_av_zone, self.config.get('keypair'),
1132                                                                 userdata, config_drive, block_device_mapping))
1133             server = self.nova.servers.create(name, image_id, flavor_id, nics=net_list_vim,
1134                                               security_groups=security_groups,
1135                                               availability_zone=vm_av_zone,
1136                                               key_name=self.config.get('keypair'),
1137                                               userdata=userdata,
1138                                               config_drive=config_drive,
1139                                               block_device_mapping=block_device_mapping
1140                                               )  # , description=description)
1141
1142             vm_start_time = time.time()
1143             # Previously mentioned workaround to wait until the VM is active and then disable the port-security
1144             if no_secured_ports:
1145                 self.__wait_for_vm(server.id, 'ACTIVE')
1146
1147             for port_id in no_secured_ports:
1148                 try:
1149                     self.neutron.update_port(port_id,
1150                                              {"port": {"port_security_enabled": False, "security_groups": None}})
1151                 except Exception as e:
1152                     raise vimconn.vimconnException("It was not possible to disable port security for port {}".format(
1153                         port_id))
1154             # print "DONE :-)", server
1155
1156             # pool_id = None
1157             if external_network:
1158                 floating_ips = self.neutron.list_floatingips().get("floatingips", ())
1159             for floating_network in external_network:
1160                 try:
1161                     assigned = False
1162                     while not assigned:
1163                         if floating_ips:
1164                             ip = floating_ips.pop(0)
1165                             if ip.get("port_id", False) or ip.get('tenant_id') != server.tenant_id:
1166                                 continue
1167                             if isinstance(floating_network['floating_ip'], str):
1168                                 if ip.get("floating_network_id") != floating_network['floating_ip']:
1169                                     continue
1170                             free_floating_ip = ip.get("floating_ip_address")
1171                         else:
1172                             if isinstance(floating_network['floating_ip'], str) and \
1173                                 floating_network['floating_ip'].lower() != "true":
1174                                 pool_id = floating_network['floating_ip']
1175                             else:
1176                                 # Find the external network
1177                                 external_nets = list()
1178                                 for net in self.neutron.list_networks()['networks']:
1179                                     if net['router:external']:
1180                                             external_nets.append(net)
1181
1182                                 if len(external_nets) == 0:
1183                                     raise vimconn.vimconnException("Cannot create floating_ip automatically since no external "
1184                                                                    "network is present",
1185                                                                     http_code=vimconn.HTTP_Conflict)
1186                                 if len(external_nets) > 1:
1187                                     raise vimconn.vimconnException("Cannot create floating_ip automatically since multiple "
1188                                                                    "external networks are present",
1189                                                                    http_code=vimconn.HTTP_Conflict)
1190
1191                                 pool_id = external_nets[0].get('id')
1192                             param = {'floatingip': {'floating_network_id': pool_id, 'tenant_id': server.tenant_id}}
1193                             try:
1194                                 # self.logger.debug("Creating floating IP")
1195                                 new_floating_ip = self.neutron.create_floatingip(param)
1196                                 free_floating_ip = new_floating_ip['floatingip']['floating_ip_address']
1197                             except Exception as e:
1198                                 raise vimconn.vimconnException(type(e).__name__ + ": Cannot create new floating_ip " +
1199                                                                str(e), http_code=vimconn.HTTP_Conflict)
1200
1201                         fix_ip = floating_network.get('ip')
1202                         while not assigned:
1203                             try:
1204                                 server.add_floating_ip(free_floating_ip, fix_ip)
1205                                 assigned = True
1206                             except Exception as e:
1207                                 # openstack need some time after VM creation to asign an IP. So retry if fails
1208                                 vm_status = self.nova.servers.get(server.id).status
1209                                 if vm_status != 'ACTIVE' and vm_status != 'ERROR':
1210                                     if time.time() - vm_start_time < server_timeout:
1211                                         time.sleep(5)
1212                                         continue
1213                                 raise vimconn.vimconnException(
1214                                     "Cannot create floating_ip: {} {}".format(type(e).__name__, e),
1215                                     http_code=vimconn.HTTP_Conflict)
1216
1217                 except Exception as e:
1218                     if not floating_network['exit_on_floating_ip_error']:
1219                         self.logger.warn("Cannot create floating_ip. %s", str(e))
1220                         continue
1221                     raise
1222
1223             return server.id, created_items
1224 #        except nvExceptions.NotFound as e:
1225 #            error_value=-vimconn.HTTP_Not_Found
1226 #            error_text= "vm instance %s not found" % vm_id
1227 #        except TypeError as e:
1228 #            raise vimconn.vimconnException(type(e).__name__ + ": "+  str(e), http_code=vimconn.HTTP_Bad_Request)
1229
1230         except Exception as e:
1231             server_id = None
1232             if server:
1233                 server_id = server.id
1234             try:
1235                 self.delete_vminstance(server_id, created_items)
1236             except Exception as e2:
1237                 self.logger.error("new_vminstance rollback fail {}".format(e2))
1238
1239             self._format_exception(e)
1240
1241     def get_vminstance(self,vm_id):
1242         '''Returns the VM instance information from VIM'''
1243         #self.logger.debug("Getting VM from VIM")
1244         try:
1245             self._reload_connection()
1246             server = self.nova.servers.find(id=vm_id)
1247             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
1248             return server.to_dict()
1249         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1250             self._format_exception(e)
1251
1252     def get_vminstance_console(self,vm_id, console_type="vnc"):
1253         '''
1254         Get a console for the virtual machine
1255         Params:
1256             vm_id: uuid of the VM
1257             console_type, can be:
1258                 "novnc" (by default), "xvpvnc" for VNC types,
1259                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
1260         Returns dict with the console parameters:
1261                 protocol: ssh, ftp, http, https, ...
1262                 server:   usually ip address
1263                 port:     the http, ssh, ... port
1264                 suffix:   extra text, e.g. the http path and query string
1265         '''
1266         self.logger.debug("Getting VM CONSOLE from VIM")
1267         try:
1268             self._reload_connection()
1269             server = self.nova.servers.find(id=vm_id)
1270             if console_type == None or console_type == "novnc":
1271                 console_dict = server.get_vnc_console("novnc")
1272             elif console_type == "xvpvnc":
1273                 console_dict = server.get_vnc_console(console_type)
1274             elif console_type == "rdp-html5":
1275                 console_dict = server.get_rdp_console(console_type)
1276             elif console_type == "spice-html5":
1277                 console_dict = server.get_spice_console(console_type)
1278             else:
1279                 raise vimconn.vimconnException("console type '{}' not allowed".format(console_type), http_code=vimconn.HTTP_Bad_Request)
1280
1281             console_dict1 = console_dict.get("console")
1282             if console_dict1:
1283                 console_url = console_dict1.get("url")
1284                 if console_url:
1285                     #parse console_url
1286                     protocol_index = console_url.find("//")
1287                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1288                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1289                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1290                         return -vimconn.HTTP_Internal_Server_Error, "Unexpected response from VIM"
1291                     console_dict={"protocol": console_url[0:protocol_index],
1292                                   "server":   console_url[protocol_index+2:port_index],
1293                                   "port":     console_url[port_index:suffix_index],
1294                                   "suffix":   console_url[suffix_index+1:]
1295                                   }
1296                     protocol_index += 2
1297                     return console_dict
1298             raise vimconn.vimconnUnexpectedResponse("Unexpected response from VIM")
1299
1300         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.BadRequest, ConnectionError) as e:
1301             self._format_exception(e)
1302
1303     def delete_vminstance(self, vm_id, created_items=None):
1304         '''Removes a VM instance from VIM. Returns the old identifier
1305         '''
1306         #print "osconnector: Getting VM from VIM"
1307         if created_items == None:
1308             created_items = {}
1309         try:
1310             self._reload_connection()
1311             # delete VM ports attached to this networks before the virtual machine
1312             for k, v in created_items.items():
1313                 if not v:  # skip already deleted
1314                     continue
1315                 try:
1316                     k_item, _, k_id = k.partition(":")
1317                     if k_item == "port":
1318                         self.neutron.delete_port(k_id)
1319                 except Exception as e:
1320                     self.logger.error("Error deleting port: {}: {}".format(type(e).__name__, e))
1321
1322             # #commented because detaching the volumes makes the servers.delete not work properly ?!?
1323             # #dettach volumes attached
1324             # server = self.nova.servers.get(vm_id)
1325             # volumes_attached_dict = server._info['os-extended-volumes:volumes_attached']   #volume['id']
1326             # #for volume in volumes_attached_dict:
1327             # #    self.cinder.volumes.detach(volume['id'])
1328
1329             if vm_id:
1330                 self.nova.servers.delete(vm_id)
1331
1332             # delete volumes. Although having detached, they should have in active status before deleting
1333             # we ensure in this loop
1334             keep_waiting = True
1335             elapsed_time = 0
1336             while keep_waiting and elapsed_time < volume_timeout:
1337                 keep_waiting = False
1338                 for k, v in created_items.items():
1339                     if not v:  # skip already deleted
1340                         continue
1341                     try:
1342                         k_item, _, k_id = k.partition(":")
1343                         if k_item == "volume":
1344                             if self.cinder.volumes.get(k_id).status != 'available':
1345                                 keep_waiting = True
1346                             else:
1347                                 self.cinder.volumes.delete(k_id)
1348                     except Exception as e:
1349                         self.logger.error("Error deleting volume: {}: {}".format(type(e).__name__, e))
1350                 if keep_waiting:
1351                     time.sleep(1)
1352                     elapsed_time += 1
1353             return None
1354         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
1355             self._format_exception(e)
1356
1357     def refresh_vms_status(self, vm_list):
1358         '''Get the status of the virtual machines and their interfaces/ports
1359            Params: the list of VM identifiers
1360            Returns a dictionary with:
1361                 vm_id:          #VIM id of this Virtual Machine
1362                     status:     #Mandatory. Text with one of:
1363                                 #  DELETED (not found at vim)
1364                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1365                                 #  OTHER (Vim reported other status not understood)
1366                                 #  ERROR (VIM indicates an ERROR status)
1367                                 #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
1368                                 #  CREATING (on building process), ERROR
1369                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
1370                                 #
1371                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
1372                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
1373                     interfaces:
1374                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
1375                         mac_address:      #Text format XX:XX:XX:XX:XX:XX
1376                         vim_net_id:       #network id where this interface is connected
1377                         vim_interface_id: #interface/port VIM id
1378                         ip_address:       #null, or text with IPv4, IPv6 address
1379                         compute_node:     #identification of compute node where PF,VF interface is allocated
1380                         pci:              #PCI address of the NIC that hosts the PF,VF
1381                         vlan:             #physical VLAN used for VF
1382         '''
1383         vm_dict={}
1384         self.logger.debug("refresh_vms status: Getting tenant VM instance information from VIM")
1385         for vm_id in vm_list:
1386             vm={}
1387             try:
1388                 vm_vim = self.get_vminstance(vm_id)
1389                 if vm_vim['status'] in vmStatus2manoFormat:
1390                     vm['status']    =  vmStatus2manoFormat[ vm_vim['status'] ]
1391                 else:
1392                     vm['status']    = "OTHER"
1393                     vm['error_msg'] = "VIM status reported " + vm_vim['status']
1394                 try:
1395                     vm['vim_info']  = yaml.safe_dump(vm_vim, default_flow_style=True, width=256)
1396                 except yaml.representer.RepresenterError:
1397                     vm['vim_info'] = str(vm_vim)
1398                 vm["interfaces"] = []
1399                 if vm_vim.get('fault'):
1400                     vm['error_msg'] = str(vm_vim['fault'])
1401                 #get interfaces
1402                 try:
1403                     self._reload_connection()
1404                     port_dict = self.neutron.list_ports(device_id=vm_id)
1405                     for port in port_dict["ports"]:
1406                         interface={}
1407                         try:
1408                             interface['vim_info'] = yaml.safe_dump(port, default_flow_style=True, width=256)
1409                         except yaml.representer.RepresenterError:
1410                             interface['vim_info'] = str(port)
1411                         interface["mac_address"] = port.get("mac_address")
1412                         interface["vim_net_id"] = port["network_id"]
1413                         interface["vim_interface_id"] = port["id"]
1414                         # check if OS-EXT-SRV-ATTR:host is there,
1415                         # in case of non-admin credentials, it will be missing
1416                         if vm_vim.get('OS-EXT-SRV-ATTR:host'):
1417                             interface["compute_node"] = vm_vim['OS-EXT-SRV-ATTR:host']
1418                         interface["pci"] = None
1419
1420                         # check if binding:profile is there,
1421                         # in case of non-admin credentials, it will be missing
1422                         if port.get('binding:profile'):
1423                             if port['binding:profile'].get('pci_slot'):
1424                                 # TODO: At the moment sr-iov pci addresses are converted to PF pci addresses by setting the slot to 0x00
1425                                 # TODO: This is just a workaround valid for niantinc. Find a better way to do so
1426                                 #   CHANGE DDDD:BB:SS.F to DDDD:BB:00.(F%2)   assuming there are 2 ports per nic
1427                                 pci = port['binding:profile']['pci_slot']
1428                                 # interface["pci"] = pci[:-4] + "00." + str(int(pci[-1]) % 2)
1429                                 interface["pci"] = pci
1430                         interface["vlan"] = None
1431                         #if network is of type vlan and port is of type direct (sr-iov) then set vlan id
1432                         network = self.neutron.show_network(port["network_id"])
1433                         if network['network'].get('provider:network_type') == 'vlan' and \
1434                             port.get("binding:vnic_type") == "direct":
1435                             interface["vlan"] = network['network'].get('provider:segmentation_id')
1436                         ips=[]
1437                         #look for floating ip address
1438                         try:
1439                             floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
1440                             if floating_ip_dict.get("floatingips"):
1441                                 ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address") )
1442                         except Exception:
1443                             pass
1444
1445                         for subnet in port["fixed_ips"]:
1446                             ips.append(subnet["ip_address"])
1447                         interface["ip_address"] = ";".join(ips)
1448                         vm["interfaces"].append(interface)
1449                 except Exception as e:
1450                     self.logger.error("Error getting vm interface information {}: {}".format(type(e).__name__, e),
1451                                       exc_info=True)
1452             except vimconn.vimconnNotFoundException as e:
1453                 self.logger.error("Exception getting vm status: %s", str(e))
1454                 vm['status'] = "DELETED"
1455                 vm['error_msg'] = str(e)
1456             except vimconn.vimconnException as e:
1457                 self.logger.error("Exception getting vm status: %s", str(e))
1458                 vm['status'] = "VIM_ERROR"
1459                 vm['error_msg'] = str(e)
1460             vm_dict[vm_id] = vm
1461         return vm_dict
1462
1463     def action_vminstance(self, vm_id, action_dict, created_items={}):
1464         '''Send and action over a VM instance from VIM
1465         Returns None or the console dict if the action was successfully sent to the VIM'''
1466         self.logger.debug("Action over VM '%s': %s", vm_id, str(action_dict))
1467         try:
1468             self._reload_connection()
1469             server = self.nova.servers.find(id=vm_id)
1470             if "start" in action_dict:
1471                 if action_dict["start"]=="rebuild":
1472                     server.rebuild()
1473                 else:
1474                     if server.status=="PAUSED":
1475                         server.unpause()
1476                     elif server.status=="SUSPENDED":
1477                         server.resume()
1478                     elif server.status=="SHUTOFF":
1479                         server.start()
1480             elif "pause" in action_dict:
1481                 server.pause()
1482             elif "resume" in action_dict:
1483                 server.resume()
1484             elif "shutoff" in action_dict or "shutdown" in action_dict:
1485                 server.stop()
1486             elif "forceOff" in action_dict:
1487                 server.stop() #TODO
1488             elif "terminate" in action_dict:
1489                 server.delete()
1490             elif "createImage" in action_dict:
1491                 server.create_image()
1492                 #"path":path_schema,
1493                 #"description":description_schema,
1494                 #"name":name_schema,
1495                 #"metadata":metadata_schema,
1496                 #"imageRef": id_schema,
1497                 #"disk": {"oneOf":[{"type": "null"}, {"type":"string"}] },
1498             elif "rebuild" in action_dict:
1499                 server.rebuild(server.image['id'])
1500             elif "reboot" in action_dict:
1501                 server.reboot() #reboot_type='SOFT'
1502             elif "console" in action_dict:
1503                 console_type = action_dict["console"]
1504                 if console_type == None or console_type == "novnc":
1505                     console_dict = server.get_vnc_console("novnc")
1506                 elif console_type == "xvpvnc":
1507                     console_dict = server.get_vnc_console(console_type)
1508                 elif console_type == "rdp-html5":
1509                     console_dict = server.get_rdp_console(console_type)
1510                 elif console_type == "spice-html5":
1511                     console_dict = server.get_spice_console(console_type)
1512                 else:
1513                     raise vimconn.vimconnException("console type '{}' not allowed".format(console_type),
1514                                                    http_code=vimconn.HTTP_Bad_Request)
1515                 try:
1516                     console_url = console_dict["console"]["url"]
1517                     #parse console_url
1518                     protocol_index = console_url.find("//")
1519                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1520                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1521                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1522                         raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1523                     console_dict2={"protocol": console_url[0:protocol_index],
1524                                   "server":   console_url[protocol_index+2 : port_index],
1525                                   "port":     int(console_url[port_index+1 : suffix_index]),
1526                                   "suffix":   console_url[suffix_index+1:]
1527                                   }
1528                     return console_dict2
1529                 except Exception as e:
1530                     raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1531
1532             return None
1533         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1534             self._format_exception(e)
1535         #TODO insert exception vimconn.HTTP_Unauthorized
1536
1537     ####### VIO Specific Changes #########
1538     def _genrate_vlanID(self):
1539         """
1540          Method to get unused vlanID
1541             Args:
1542                 None
1543             Returns:
1544                 vlanID
1545         """
1546         #Get used VLAN IDs
1547         usedVlanIDs = []
1548         networks = self.get_network_list()
1549         for net in networks:
1550             if net.get('provider:segmentation_id'):
1551                 usedVlanIDs.append(net.get('provider:segmentation_id'))
1552         used_vlanIDs = set(usedVlanIDs)
1553
1554         #find unused VLAN ID
1555         for vlanID_range in self.config.get('dataplane_net_vlan_range'):
1556             try:
1557                 start_vlanid , end_vlanid = map(int, vlanID_range.replace(" ", "").split("-"))
1558                 for vlanID in xrange(start_vlanid, end_vlanid + 1):
1559                     if vlanID not in used_vlanIDs:
1560                         return vlanID
1561             except Exception as exp:
1562                 raise vimconn.vimconnException("Exception {} occurred while generating VLAN ID.".format(exp))
1563         else:
1564             raise vimconn.vimconnConflictException("Unable to create the SRIOV VLAN network."\
1565                 " All given Vlan IDs {} are in use.".format(self.config.get('dataplane_net_vlan_range')))
1566
1567
1568     def _validate_vlan_ranges(self, dataplane_net_vlan_range):
1569         """
1570         Method to validate user given vlanID ranges
1571             Args:  None
1572             Returns: None
1573         """
1574         for vlanID_range in dataplane_net_vlan_range:
1575             vlan_range = vlanID_range.replace(" ", "")
1576             #validate format
1577             vlanID_pattern = r'(\d)*-(\d)*$'
1578             match_obj = re.match(vlanID_pattern, vlan_range)
1579             if not match_obj:
1580                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}.You must provide "\
1581                 "'dataplane_net_vlan_range' in format [start_ID - end_ID].".format(vlanID_range))
1582
1583             start_vlanid , end_vlanid = map(int,vlan_range.split("-"))
1584             if start_vlanid <= 0 :
1585                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1586                 "Start ID can not be zero. For VLAN "\
1587                 "networks valid IDs are 1 to 4094 ".format(vlanID_range))
1588             if end_vlanid > 4094 :
1589                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1590                 "End VLAN ID can not be greater than 4094. For VLAN "\
1591                 "networks valid IDs are 1 to 4094 ".format(vlanID_range))
1592
1593             if start_vlanid > end_vlanid:
1594                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1595                     "You must provide a 'dataplane_net_vlan_range' in format start_ID - end_ID and "\
1596                     "start_ID < end_ID ".format(vlanID_range))
1597
1598 #NOT USED FUNCTIONS
1599
1600     def new_external_port(self, port_data):
1601         #TODO openstack if needed
1602         '''Adds a external port to VIM'''
1603         '''Returns the port identifier'''
1604         return -vimconn.HTTP_Internal_Server_Error, "osconnector.new_external_port() not implemented"
1605
1606     def connect_port_network(self, port_id, network_id, admin=False):
1607         #TODO openstack if needed
1608         '''Connects a external port to a network'''
1609         '''Returns status code of the VIM response'''
1610         return -vimconn.HTTP_Internal_Server_Error, "osconnector.connect_port_network() not implemented"
1611
1612     def new_user(self, user_name, user_passwd, tenant_id=None):
1613         '''Adds a new user to openstack VIM'''
1614         '''Returns the user identifier'''
1615         self.logger.debug("osconnector: Adding a new user to VIM")
1616         try:
1617             self._reload_connection()
1618             user=self.keystone.users.create(user_name, user_passwd, tenant_id=tenant_id)
1619             #self.keystone.tenants.add_user(self.k_creds["username"], #role)
1620             return user.id
1621         except ksExceptions.ConnectionError as e:
1622             error_value=-vimconn.HTTP_Bad_Request
1623             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1624         except ksExceptions.ClientException as e: #TODO remove
1625             error_value=-vimconn.HTTP_Bad_Request
1626             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1627         #TODO insert exception vimconn.HTTP_Unauthorized
1628         #if reaching here is because an exception
1629         self.logger.debug("new_user " + error_text)
1630         return error_value, error_text
1631
1632     def delete_user(self, user_id):
1633         '''Delete a user from openstack VIM'''
1634         '''Returns the user identifier'''
1635         if self.debug:
1636             print("osconnector: Deleting  a  user from VIM")
1637         try:
1638             self._reload_connection()
1639             self.keystone.users.delete(user_id)
1640             return 1, user_id
1641         except ksExceptions.ConnectionError as e:
1642             error_value=-vimconn.HTTP_Bad_Request
1643             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1644         except ksExceptions.NotFound as e:
1645             error_value=-vimconn.HTTP_Not_Found
1646             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1647         except ksExceptions.ClientException as e: #TODO remove
1648             error_value=-vimconn.HTTP_Bad_Request
1649             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1650         #TODO insert exception vimconn.HTTP_Unauthorized
1651         #if reaching here is because an exception
1652             self.logger.debug("delete_tenant " + error_text)
1653         return error_value, error_text
1654
1655     def get_hosts_info(self):
1656         '''Get the information of deployed hosts
1657         Returns the hosts content'''
1658         if self.debug:
1659             print("osconnector: Getting Host info from VIM")
1660         try:
1661             h_list=[]
1662             self._reload_connection()
1663             hypervisors = self.nova.hypervisors.list()
1664             for hype in hypervisors:
1665                 h_list.append( hype.to_dict() )
1666             return 1, {"hosts":h_list}
1667         except nvExceptions.NotFound as e:
1668             error_value=-vimconn.HTTP_Not_Found
1669             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1670         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1671             error_value=-vimconn.HTTP_Bad_Request
1672             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1673         #TODO insert exception vimconn.HTTP_Unauthorized
1674         #if reaching here is because an exception
1675         self.logger.debug("get_hosts_info " + error_text)
1676         return error_value, error_text
1677
1678     def get_hosts(self, vim_tenant):
1679         '''Get the hosts and deployed instances
1680         Returns the hosts content'''
1681         r, hype_dict = self.get_hosts_info()
1682         if r<0:
1683             return r, hype_dict
1684         hypervisors = hype_dict["hosts"]
1685         try:
1686             servers = self.nova.servers.list()
1687             for hype in hypervisors:
1688                 for server in servers:
1689                     if server.to_dict()['OS-EXT-SRV-ATTR:hypervisor_hostname']==hype['hypervisor_hostname']:
1690                         if 'vm' in hype:
1691                             hype['vm'].append(server.id)
1692                         else:
1693                             hype['vm'] = [server.id]
1694             return 1, hype_dict
1695         except nvExceptions.NotFound as e:
1696             error_value=-vimconn.HTTP_Not_Found
1697             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1698         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1699             error_value=-vimconn.HTTP_Bad_Request
1700             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1701         #TODO insert exception vimconn.HTTP_Unauthorized
1702         #if reaching here is because an exception
1703         self.logger.debug("get_hosts " + error_text)
1704         return error_value, error_text
1705
1706     def new_classification(self, name, ctype, definition):
1707         self.logger.debug(
1708             'Adding a new (Traffic) Classification to VIM, named %s', name)
1709         try:
1710             new_class = None
1711             self._reload_connection()
1712             if ctype not in supportedClassificationTypes:
1713                 raise vimconn.vimconnNotSupportedException(
1714                         'OpenStack VIM connector doesn\'t support provided '
1715                         'Classification Type {}, supported ones are: '
1716                         '{}'.format(ctype, supportedClassificationTypes))
1717             if not self._validate_classification(ctype, definition):
1718                 raise vimconn.vimconnException(
1719                     'Incorrect Classification definition '
1720                     'for the type specified.')
1721             classification_dict = definition
1722             classification_dict['name'] = name
1723
1724             new_class = self.neutron.create_sfc_flow_classifier(
1725                 {'flow_classifier': classification_dict})
1726             return new_class['flow_classifier']['id']
1727         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1728                 neExceptions.NeutronException, ConnectionError) as e:
1729             self.logger.error(
1730                 'Creation of Classification failed.')
1731             self._format_exception(e)
1732
1733     def get_classification(self, class_id):
1734         self.logger.debug(" Getting Classification %s from VIM", class_id)
1735         filter_dict = {"id": class_id}
1736         class_list = self.get_classification_list(filter_dict)
1737         if len(class_list) == 0:
1738             raise vimconn.vimconnNotFoundException(
1739                 "Classification '{}' not found".format(class_id))
1740         elif len(class_list) > 1:
1741             raise vimconn.vimconnConflictException(
1742                 "Found more than one Classification with this criteria")
1743         classification = class_list[0]
1744         return classification
1745
1746     def get_classification_list(self, filter_dict={}):
1747         self.logger.debug("Getting Classifications from VIM filter: '%s'",
1748                           str(filter_dict))
1749         try:
1750             filter_dict_os = filter_dict.copy()
1751             self._reload_connection()
1752             if self.api_version3 and "tenant_id" in filter_dict_os:
1753                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1754             classification_dict = self.neutron.list_sfc_flow_classifiers(
1755                 **filter_dict_os)
1756             classification_list = classification_dict["flow_classifiers"]
1757             self.__classification_os2mano(classification_list)
1758             return classification_list
1759         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1760                 neExceptions.NeutronException, ConnectionError) as e:
1761             self._format_exception(e)
1762
1763     def delete_classification(self, class_id):
1764         self.logger.debug("Deleting Classification '%s' from VIM", class_id)
1765         try:
1766             self._reload_connection()
1767             self.neutron.delete_sfc_flow_classifier(class_id)
1768             return class_id
1769         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1770                 ksExceptions.ClientException, neExceptions.NeutronException,
1771                 ConnectionError) as e:
1772             self._format_exception(e)
1773
1774     def new_sfi(self, name, ingress_ports, egress_ports, sfc_encap=True):
1775         self.logger.debug(
1776             "Adding a new Service Function Instance to VIM, named '%s'", name)
1777         try:
1778             new_sfi = None
1779             self._reload_connection()
1780             correlation = None
1781             if sfc_encap:
1782                 correlation = 'nsh'
1783             if len(ingress_ports) != 1:
1784                 raise vimconn.vimconnNotSupportedException(
1785                     "OpenStack VIM connector can only have "
1786                     "1 ingress port per SFI")
1787             if len(egress_ports) != 1:
1788                 raise vimconn.vimconnNotSupportedException(
1789                     "OpenStack VIM connector can only have "
1790                     "1 egress port per SFI")
1791             sfi_dict = {'name': name,
1792                         'ingress': ingress_ports[0],
1793                         'egress': egress_ports[0],
1794                         'service_function_parameters': {
1795                             'correlation': correlation}}
1796             new_sfi = self.neutron.create_sfc_port_pair({'port_pair': sfi_dict})
1797             return new_sfi['port_pair']['id']
1798         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1799                 neExceptions.NeutronException, ConnectionError) as e:
1800             if new_sfi:
1801                 try:
1802                     self.neutron.delete_sfc_port_pair(
1803                         new_sfi['port_pair']['id'])
1804                 except Exception:
1805                     self.logger.error(
1806                         'Creation of Service Function Instance failed, with '
1807                         'subsequent deletion failure as well.')
1808             self._format_exception(e)
1809
1810     def get_sfi(self, sfi_id):
1811         self.logger.debug(
1812             'Getting Service Function Instance %s from VIM', sfi_id)
1813         filter_dict = {"id": sfi_id}
1814         sfi_list = self.get_sfi_list(filter_dict)
1815         if len(sfi_list) == 0:
1816             raise vimconn.vimconnNotFoundException(
1817                 "Service Function Instance '{}' not found".format(sfi_id))
1818         elif len(sfi_list) > 1:
1819             raise vimconn.vimconnConflictException(
1820                 'Found more than one Service Function Instance '
1821                 'with this criteria')
1822         sfi = sfi_list[0]
1823         return sfi
1824
1825     def get_sfi_list(self, filter_dict={}):
1826         self.logger.debug("Getting Service Function Instances from "
1827                           "VIM filter: '%s'", str(filter_dict))
1828         try:
1829             self._reload_connection()
1830             filter_dict_os = filter_dict.copy()
1831             if self.api_version3 and "tenant_id" in filter_dict_os:
1832                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1833             sfi_dict = self.neutron.list_sfc_port_pairs(**filter_dict_os)
1834             sfi_list = sfi_dict["port_pairs"]
1835             self.__sfi_os2mano(sfi_list)
1836             return sfi_list
1837         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1838                 neExceptions.NeutronException, ConnectionError) as e:
1839             self._format_exception(e)
1840
1841     def delete_sfi(self, sfi_id):
1842         self.logger.debug("Deleting Service Function Instance '%s' "
1843                           "from VIM", sfi_id)
1844         try:
1845             self._reload_connection()
1846             self.neutron.delete_sfc_port_pair(sfi_id)
1847             return sfi_id
1848         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1849                 ksExceptions.ClientException, neExceptions.NeutronException,
1850                 ConnectionError) as e:
1851             self._format_exception(e)
1852
1853     def new_sf(self, name, sfis, sfc_encap=True):
1854         self.logger.debug("Adding a new Service Function to VIM, "
1855                           "named '%s'", name)
1856         try:
1857             new_sf = None
1858             self._reload_connection()
1859             # correlation = None
1860             # if sfc_encap:
1861             #     correlation = 'nsh'
1862             for instance in sfis:
1863                 sfi = self.get_sfi(instance)
1864                 if sfi.get('sfc_encap') != sfc_encap:
1865                     raise vimconn.vimconnNotSupportedException(
1866                         "OpenStack VIM connector requires all SFIs of the "
1867                         "same SF to share the same SFC Encapsulation")
1868             sf_dict = {'name': name,
1869                        'port_pairs': sfis}
1870             new_sf = self.neutron.create_sfc_port_pair_group({
1871                 'port_pair_group': sf_dict})
1872             return new_sf['port_pair_group']['id']
1873         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1874                 neExceptions.NeutronException, ConnectionError) as e:
1875             if new_sf:
1876                 try:
1877                     self.neutron.delete_sfc_port_pair_group(
1878                         new_sf['port_pair_group']['id'])
1879                 except Exception:
1880                     self.logger.error(
1881                         'Creation of Service Function failed, with '
1882                         'subsequent deletion failure as well.')
1883             self._format_exception(e)
1884
1885     def get_sf(self, sf_id):
1886         self.logger.debug("Getting Service Function %s from VIM", sf_id)
1887         filter_dict = {"id": sf_id}
1888         sf_list = self.get_sf_list(filter_dict)
1889         if len(sf_list) == 0:
1890             raise vimconn.vimconnNotFoundException(
1891                 "Service Function '{}' not found".format(sf_id))
1892         elif len(sf_list) > 1:
1893             raise vimconn.vimconnConflictException(
1894                 "Found more than one Service Function with this criteria")
1895         sf = sf_list[0]
1896         return sf
1897
1898     def get_sf_list(self, filter_dict={}):
1899         self.logger.debug("Getting Service Function from VIM filter: '%s'",
1900                           str(filter_dict))
1901         try:
1902             self._reload_connection()
1903             filter_dict_os = filter_dict.copy()
1904             if self.api_version3 and "tenant_id" in filter_dict_os:
1905                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1906             sf_dict = self.neutron.list_sfc_port_pair_groups(**filter_dict_os)
1907             sf_list = sf_dict["port_pair_groups"]
1908             self.__sf_os2mano(sf_list)
1909             return sf_list
1910         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1911                 neExceptions.NeutronException, ConnectionError) as e:
1912             self._format_exception(e)
1913
1914     def delete_sf(self, sf_id):
1915         self.logger.debug("Deleting Service Function '%s' from VIM", sf_id)
1916         try:
1917             self._reload_connection()
1918             self.neutron.delete_sfc_port_pair_group(sf_id)
1919             return sf_id
1920         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1921                 ksExceptions.ClientException, neExceptions.NeutronException,
1922                 ConnectionError) as e:
1923             self._format_exception(e)
1924
1925     def new_sfp(self, name, classifications, sfs, sfc_encap=True, spi=None):
1926         self.logger.debug("Adding a new Service Function Path to VIM, "
1927                           "named '%s'", name)
1928         try:
1929             new_sfp = None
1930             self._reload_connection()
1931             # In networking-sfc the MPLS encapsulation is legacy
1932             # should be used when no full SFC Encapsulation is intended
1933             sfc_encap = 'mpls'
1934             if sfc_encap:
1935                 correlation = 'nsh'
1936             sfp_dict = {'name': name,
1937                         'flow_classifiers': classifications,
1938                         'port_pair_groups': sfs,
1939                         'chain_parameters': {'correlation': correlation}}
1940             if spi:
1941                 sfp_dict['chain_id'] = spi
1942             new_sfp = self.neutron.create_sfc_port_chain({'port_chain': sfp_dict})
1943             return new_sfp["port_chain"]["id"]
1944         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1945                 neExceptions.NeutronException, ConnectionError) as e:
1946             if new_sfp:
1947                 try:
1948                     self.neutron.delete_sfc_port_chain(new_sfp['port_chain']['id'])
1949                 except Exception:
1950                     self.logger.error(
1951                         'Creation of Service Function Path failed, with '
1952                         'subsequent deletion failure as well.')
1953             self._format_exception(e)
1954
1955     def get_sfp(self, sfp_id):
1956         self.logger.debug(" Getting Service Function Path %s from VIM", sfp_id)
1957         filter_dict = {"id": sfp_id}
1958         sfp_list = self.get_sfp_list(filter_dict)
1959         if len(sfp_list) == 0:
1960             raise vimconn.vimconnNotFoundException(
1961                 "Service Function Path '{}' not found".format(sfp_id))
1962         elif len(sfp_list) > 1:
1963             raise vimconn.vimconnConflictException(
1964                 "Found more than one Service Function Path with this criteria")
1965         sfp = sfp_list[0]
1966         return sfp
1967
1968     def get_sfp_list(self, filter_dict={}):
1969         self.logger.debug("Getting Service Function Paths from VIM filter: "
1970                           "'%s'", str(filter_dict))
1971         try:
1972             self._reload_connection()
1973             filter_dict_os = filter_dict.copy()
1974             if self.api_version3 and "tenant_id" in filter_dict_os:
1975                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1976             sfp_dict = self.neutron.list_sfc_port_chains(**filter_dict_os)
1977             sfp_list = sfp_dict["port_chains"]
1978             self.__sfp_os2mano(sfp_list)
1979             return sfp_list
1980         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1981                 neExceptions.NeutronException, ConnectionError) as e:
1982             self._format_exception(e)
1983
1984     def delete_sfp(self, sfp_id):
1985         self.logger.debug(
1986             "Deleting Service Function Path '%s' from VIM", sfp_id)
1987         try:
1988             self._reload_connection()
1989             self.neutron.delete_sfc_port_chain(sfp_id)
1990             return sfp_id
1991         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1992                 ksExceptions.ClientException, neExceptions.NeutronException,
1993                 ConnectionError) as e:
1994             self._format_exception(e)