osm_ro/vimconn_openstack.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 osconnector implements all the methods to interact with openstack using the python-neutronclient.
  26
  27 For the VNF forwarding graph, The OpenStack VIM connector calls the
  28 networking-sfc Neutron extension methods, whose resources are mapped
  29 to the VIM connector's SFC resources as follows:
  30 - Classification (OSM) -> Flow Classifier (Neutron)
  31 - Service Function Instance (OSM) -> Port Pair (Neutron)
  32 - Service Function (OSM) -> Port Pair Group (Neutron)
  33 - Service Function Path (OSM) -> Port Chain (Neutron)
  34 '''
  35 __author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes, xFlow Research, Igor D.C., Eduardo Sousa"
  36 __date__  = "$22-sep-2017 23:59:59$"
  37
  38 import vimconn
  39 # import json
  40 import logging
  41 import netaddr
  42 import time
  43 import yaml
  44 import random
  45 import re
  46 import copy
  47
  48 from novaclient import client as nClient, exceptions as nvExceptions
  49 from keystoneauth1.identity import v2, v3
  50 from keystoneauth1 import session
  51 import keystoneclient.exceptions as ksExceptions
  52 import keystoneclient.v3.client as ksClient_v3
  53 import keystoneclient.v2_0.client as ksClient_v2
  54 from glanceclient import client as glClient
  55 import glanceclient.exc as gl1Exceptions
  56 from  cinderclient import client as cClient
  57 from httplib import HTTPException
  58 from neutronclient.neutron import client as neClient
  59 from neutronclient.common import exceptions as neExceptions
  60 from requests.exceptions import ConnectionError
  61
  62
  63 """contain the openstack virtual machine status to openmano status"""
  64 vmStatus2manoFormat={'ACTIVE':'ACTIVE',
  65                      'PAUSED':'PAUSED',
  66                      'SUSPENDED': 'SUSPENDED',
  67                      'SHUTOFF':'INACTIVE',
  68                      'BUILD':'BUILD',
  69                      'ERROR':'ERROR','DELETED':'DELETED'
  70                      }
  71 netStatus2manoFormat={'ACTIVE':'ACTIVE','PAUSED':'PAUSED','INACTIVE':'INACTIVE','BUILD':'BUILD','ERROR':'ERROR','DELETED':'DELETED'
  72                      }
  73
  74 supportedClassificationTypes = ['legacy_flow_classifier']
  75
  76 #global var to have a timeout creating and deleting volumes
  77 volume_timeout = 600
  78 server_timeout = 600
  79
  80 class vimconnector(vimconn.vimconnector):
  81     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None,
  82                  log_level=None, config={}, persistent_info={}):
  83         '''using common constructor parameters. In this case
  84         'url' is the keystone authorization url,
  85         'url_admin' is not use
  86         '''
  87         api_version = config.get('APIversion')
  88         if api_version and api_version not in ('v3.3', 'v2.0', '2', '3'):
  89             raise vimconn.vimconnException("Invalid value '{}' for config:APIversion. "
  90                                            "Allowed values are 'v3.3', 'v2.0', '2' or '3'".format(api_version))
  91         vim_type = config.get('vim_type')
  92         if vim_type and vim_type not in ('vio', 'VIO'):
  93             raise vimconn.vimconnException("Invalid value '{}' for config:vim_type."
  94                             "Allowed values are 'vio' or 'VIO'".format(vim_type))
  95
  96         if config.get('dataplane_net_vlan_range') is not None:
  97             #validate vlan ranges provided by user
  98             self._validate_vlan_ranges(config.get('dataplane_net_vlan_range'))
  99
 100         vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level,
 101                                       config)
 102
 103         if self.config.get("insecure") and self.config.get("ca_cert"):
 104             raise vimconn.vimconnException("options insecure and ca_cert are mutually exclusive")
 105         self.verify = True
 106         if self.config.get("insecure"):
 107             self.verify = False
 108         if self.config.get("ca_cert"):
 109             self.verify = self.config.get("ca_cert")
 110
 111         if not url:
 112             raise TypeError('url param can not be NoneType')
 113         self.persistent_info = persistent_info
 114         self.availability_zone = persistent_info.get('availability_zone', None)
 115         self.session = persistent_info.get('session', {'reload_client': True})
 116         self.nova = self.session.get('nova')
 117         self.neutron = self.session.get('neutron')
 118         self.cinder = self.session.get('cinder')
 119         self.glance = self.session.get('glance')
 120         # self.glancev1 = self.session.get('glancev1')
 121         self.keystone = self.session.get('keystone')
 122         self.api_version3 = self.session.get('api_version3')
 123         self.vim_type = self.config.get("vim_type")
 124         if self.vim_type:
 125             self.vim_type = self.vim_type.upper()
 126         if self.config.get("use_internal_endpoint"):
 127             self.endpoint_type = "internalURL"
 128         else:
 129             self.endpoint_type = None
 130
 131         self.logger = logging.getLogger('openmano.vim.openstack')
 132
 133         ####### VIO Specific Changes #########
 134         if self.vim_type == "VIO":
 135             self.logger = logging.getLogger('openmano.vim.vio')
 136
 137         if log_level:
 138             self.logger.setLevel( getattr(logging, log_level))
 139
 140     def __getitem__(self, index):
 141         """Get individuals parameters.
 142         Throw KeyError"""
 143         if index == 'project_domain_id':
 144             return self.config.get("project_domain_id")
 145         elif index == 'user_domain_id':
 146             return self.config.get("user_domain_id")
 147         else:
 148             return vimconn.vimconnector.__getitem__(self, index)
 149
 150     def __setitem__(self, index, value):
 151         """Set individuals parameters and it is marked as dirty so to force connection reload.
 152         Throw KeyError"""
 153         if index == 'project_domain_id':
 154             self.config["project_domain_id"] = value
 155         elif index == 'user_domain_id':
 156                 self.config["user_domain_id"] = value
 157         else:
 158             vimconn.vimconnector.__setitem__(self, index, value)
 159         self.session['reload_client'] = True
 160
 161     def _reload_connection(self):
 162         '''Called before any operation, it check if credentials has changed
 163         Throw keystoneclient.apiclient.exceptions.AuthorizationFailure
 164         '''
 165         #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-)
 166         if self.session['reload_client']:
 167             if self.config.get('APIversion'):
 168                 self.api_version3 = self.config['APIversion'] == 'v3.3' or self.config['APIversion'] == '3'
 169             else:  # get from ending auth_url that end with v3 or with v2.0
 170                 self.api_version3 =  self.url.endswith("/v3") or self.url.endswith("/v3/")
 171             self.session['api_version3'] = self.api_version3
 172             if self.api_version3:
 173                 if self.config.get('project_domain_id') or self.config.get('project_domain_name'):
 174                     project_domain_id_default = None
 175                 else:
 176                     project_domain_id_default = 'default'
 177                 if self.config.get('user_domain_id') or self.config.get('user_domain_name'):
 178                     user_domain_id_default = None
 179                 else:
 180                     user_domain_id_default = 'default'
 181                 auth = v3.Password(auth_url=self.url,
 182                                    username=self.user,
 183                                    password=self.passwd,
 184                                    project_name=self.tenant_name,
 185                                    project_id=self.tenant_id,
 186                                    project_domain_id=self.config.get('project_domain_id', project_domain_id_default),
 187                                    user_domain_id=self.config.get('user_domain_id', user_domain_id_default),
 188                                    project_domain_name=self.config.get('project_domain_name'),
 189                                    user_domain_name=self.config.get('user_domain_name'))
 190             else:
 191                 auth = v2.Password(auth_url=self.url,
 192                                    username=self.user,
 193                                    password=self.passwd,
 194                                    tenant_name=self.tenant_name,
 195                                    tenant_id=self.tenant_id)
 196             sess = session.Session(auth=auth, verify=self.verify)
 197             if self.api_version3:
 198                 self.keystone = ksClient_v3.Client(session=sess, endpoint_type=self.endpoint_type)
 199             else:
 200                 self.keystone = ksClient_v2.Client(session=sess, endpoint_type=self.endpoint_type)
 201             self.session['keystone'] = self.keystone
 202             # In order to enable microversion functionality an explicit microversion must be specified in 'config'.
 203             # This implementation approach is due to the warning message in
 204             # https://developer.openstack.org/api-guide/compute/microversions.html
 205             # where it is stated that microversion backwards compatibility is not guaranteed and clients should
 206             # always require an specific microversion.
 207             # To be able to use 'device role tagging' functionality define 'microversion: 2.32' in datacenter config
 208             version = self.config.get("microversion")
 209             if not version:
 210                 version = "2.1"
 211             self.nova = self.session['nova'] = nClient.Client(str(version), session=sess, endpoint_type=self.endpoint_type)
 212             self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess, endpoint_type=self.endpoint_type)
 213             self.cinder = self.session['cinder'] = cClient.Client(2, session=sess, endpoint_type=self.endpoint_type)
 214             if self.endpoint_type == "internalURL":
 215                 glance_service_id = self.keystone.services.list(name="glance")[0].id
 216                 glance_endpoint = self.keystone.endpoints.list(glance_service_id, interface="internal")[0].url
 217             else:
 218                 glance_endpoint = None
 219             self.glance = self.session['glance'] = glClient.Client(2, session=sess, endpoint=glance_endpoint)
 220             #using version 1 of glance client in new_image()
 221             # self.glancev1 = self.session['glancev1'] = glClient.Client('1', session=sess,
 222             #                                                            endpoint=glance_endpoint)
 223             self.session['reload_client'] = False
 224             self.persistent_info['session'] = self.session
 225             # add availablity zone info inside  self.persistent_info
 226             self._set_availablity_zones()
 227             self.persistent_info['availability_zone'] = self.availability_zone
 228
 229     def __net_os2mano(self, net_list_dict):
 230         '''Transform the net openstack format to mano format
 231         net_list_dict can be a list of dict or a single dict'''
 232         if type(net_list_dict) is dict:
 233             net_list_=(net_list_dict,)
 234         elif type(net_list_dict) is list:
 235             net_list_=net_list_dict
 236         else:
 237             raise TypeError("param net_list_dict must be a list or a dictionary")
 238         for net in net_list_:
 239             if net.get('provider:network_type') == "vlan":
 240                 net['type']='data'
 241             else:
 242                 net['type']='bridge'
 243
 244     def __classification_os2mano(self, class_list_dict):
 245         """Transform the openstack format (Flow Classifier) to mano format
 246         (Classification) class_list_dict can be a list of dict or a single dict
 247         """
 248         if isinstance(class_list_dict, dict):
 249             class_list_ = [class_list_dict]
 250         elif isinstance(class_list_dict, list):
 251             class_list_ = class_list_dict
 252         else:
 253             raise TypeError(
 254                 "param class_list_dict must be a list or a dictionary")
 255         for classification in class_list_:
 256             id = classification.pop('id')
 257             name = classification.pop('name')
 258             description = classification.pop('description')
 259             project_id = classification.pop('project_id')
 260             tenant_id = classification.pop('tenant_id')
 261             original_classification = copy.deepcopy(classification)
 262             classification.clear()
 263             classification['ctype'] = 'legacy_flow_classifier'
 264             classification['definition'] = original_classification
 265             classification['id'] = id
 266             classification['name'] = name
 267             classification['description'] = description
 268             classification['project_id'] = project_id
 269             classification['tenant_id'] = tenant_id
 270
 271     def __sfi_os2mano(self, sfi_list_dict):
 272         """Transform the openstack format (Port Pair) to mano format (SFI)
 273         sfi_list_dict can be a list of dict or a single dict
 274         """
 275         if isinstance(sfi_list_dict, dict):
 276             sfi_list_ = [sfi_list_dict]
 277         elif isinstance(sfi_list_dict, list):
 278             sfi_list_ = sfi_list_dict
 279         else:
 280             raise TypeError(
 281                 "param sfi_list_dict must be a list or a dictionary")
 282         for sfi in sfi_list_:
 283             sfi['ingress_ports'] = []
 284             sfi['egress_ports'] = []
 285             if sfi.get('ingress'):
 286                 sfi['ingress_ports'].append(sfi['ingress'])
 287             if sfi.get('egress'):
 288                 sfi['egress_ports'].append(sfi['egress'])
 289             del sfi['ingress']
 290             del sfi['egress']
 291             params = sfi.get('service_function_parameters')
 292             sfc_encap = False
 293             if params:
 294                 correlation = params.get('correlation')
 295                 if correlation:
 296                     sfc_encap = True
 297             sfi['sfc_encap'] = sfc_encap
 298             del sfi['service_function_parameters']
 299
 300     def __sf_os2mano(self, sf_list_dict):
 301         """Transform the openstack format (Port Pair Group) to mano format (SF)
 302         sf_list_dict can be a list of dict or a single dict
 303         """
 304         if isinstance(sf_list_dict, dict):
 305             sf_list_ = [sf_list_dict]
 306         elif isinstance(sf_list_dict, list):
 307             sf_list_ = sf_list_dict
 308         else:
 309             raise TypeError(
 310                 "param sf_list_dict must be a list or a dictionary")
 311         for sf in sf_list_:
 312             del sf['port_pair_group_parameters']
 313             sf['sfis'] = sf['port_pairs']
 314             del sf['port_pairs']
 315
 316     def __sfp_os2mano(self, sfp_list_dict):
 317         """Transform the openstack format (Port Chain) to mano format (SFP)
 318         sfp_list_dict can be a list of dict or a single dict
 319         """
 320         if isinstance(sfp_list_dict, dict):
 321             sfp_list_ = [sfp_list_dict]
 322         elif isinstance(sfp_list_dict, list):
 323             sfp_list_ = sfp_list_dict
 324         else:
 325             raise TypeError(
 326                 "param sfp_list_dict must be a list or a dictionary")
 327         for sfp in sfp_list_:
 328             params = sfp.pop('chain_parameters')
 329             sfc_encap = False
 330             if params:
 331                 correlation = params.get('correlation')
 332                 if correlation:
 333                     sfc_encap = True
 334             sfp['sfc_encap'] = sfc_encap
 335             sfp['spi'] = sfp.pop('chain_id')
 336             sfp['classifications'] = sfp.pop('flow_classifiers')
 337             sfp['service_functions'] = sfp.pop('port_pair_groups')
 338
 339     # placeholder for now; read TODO note below
 340     def _validate_classification(self, type, definition):
 341         # only legacy_flow_classifier Type is supported at this point
 342         return True
 343         # TODO(igordcard): this method should be an abstract method of an
 344         # abstract Classification class to be implemented by the specific
 345         # Types. Also, abstract vimconnector should call the validation
 346         # method before the implemented VIM connectors are called.
 347
 348     def _format_exception(self, exception):
 349         '''Transform a keystone, nova, neutron  exception into a vimconn exception'''
 350         if isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound, ksExceptions.NotFound, gl1Exceptions.HTTPNotFound)):
 351             raise vimconn.vimconnNotFoundException(type(exception).__name__ + ": " + str(exception))
 352         elif isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError,
 353                                ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed)):
 354             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 355         elif isinstance(exception,  (KeyError, nvExceptions.BadRequest, ksExceptions.BadRequest)):
 356             raise vimconn.vimconnException(type(exception).__name__ + ": " + str(exception))
 357         elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException,
 358                                     neExceptions.NeutronException)):
 359             raise vimconn.vimconnUnexpectedResponse(type(exception).__name__ + ": " + str(exception))
 360         elif isinstance(exception, nvExceptions.Conflict):
 361             raise vimconn.vimconnConflictException(type(exception).__name__ + ": " + str(exception))
 362         elif isinstance(exception, vimconn.vimconnException):
 363             raise exception
 364         else:  # ()
 365             self.logger.error("General Exception " + str(exception), exc_info=True)
 366             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 367
 368     def get_tenant_list(self, filter_dict={}):
 369         '''Obtain tenants of VIM
 370         filter_dict can contain the following keys:
 371             name: filter by tenant name
 372             id: filter by tenant uuid/id
 373             <other VIM specific>
 374         Returns the tenant list of dictionaries: [{'name':'<name>, 'id':'<id>, ...}, ...]
 375         '''
 376         self.logger.debug("Getting tenants from VIM filter: '%s'", str(filter_dict))
 377         try:
 378             self._reload_connection()
 379             if self.api_version3:
 380                 project_class_list = self.keystone.projects.list(name=filter_dict.get("name"))
 381             else:
 382                 project_class_list = self.keystone.tenants.findall(**filter_dict)
 383             project_list=[]
 384             for project in project_class_list:
 385                 if filter_dict.get('id') and filter_dict["id"] != project.id:
 386                     continue
 387                 project_list.append(project.to_dict())
 388             return project_list
 389         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError) as e:
 390             self._format_exception(e)
 391
 392     def new_tenant(self, tenant_name, tenant_description):
 393         '''Adds a new tenant to openstack VIM. Returns the tenant identifier'''
 394         self.logger.debug("Adding a new tenant name: %s", tenant_name)
 395         try:
 396             self._reload_connection()
 397             if self.api_version3:
 398                 project = self.keystone.projects.create(tenant_name, self.config.get("project_domain_id", "default"),
 399                                                         description=tenant_description, is_domain=False)
 400             else:
 401                 project = self.keystone.tenants.create(tenant_name, tenant_description)
 402             return project.id
 403         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ksExceptions.BadRequest, ConnectionError)  as e:
 404             self._format_exception(e)
 405
 406     def delete_tenant(self, tenant_id):
 407         '''Delete a tenant from openstack VIM. Returns the old tenant identifier'''
 408         self.logger.debug("Deleting tenant %s from VIM", tenant_id)
 409         try:
 410             self._reload_connection()
 411             if self.api_version3:
 412                 self.keystone.projects.delete(tenant_id)
 413             else:
 414                 self.keystone.tenants.delete(tenant_id)
 415             return tenant_id
 416         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ksExceptions.NotFound, ConnectionError)  as e:
 417             self._format_exception(e)
 418
 419     def new_network(self,net_name, net_type, ip_profile=None, shared=False, vlan=None):
 420         '''Adds a tenant network to VIM. Returns the network identifier'''
 421         self.logger.debug("Adding a new network to VIM name '%s', type '%s'", net_name, net_type)
 422         #self.logger.debug(">>>>>>>>>>>>>>>>>> IP profile %s", str(ip_profile))
 423         try:
 424             new_net = None
 425             self._reload_connection()
 426             network_dict = {'name': net_name, 'admin_state_up': True}
 427             if net_type=="data" or net_type=="ptp":
 428                 if self.config.get('dataplane_physical_net') == None:
 429                     raise vimconn.vimconnConflictException("You must provide a 'dataplane_physical_net' at config value before creating sriov network")
 430                 network_dict["provider:physical_network"] = self.config['dataplane_physical_net'] #"physnet_sriov" #TODO physical
 431                 network_dict["provider:network_type"]     = "vlan"
 432                 if vlan!=None:
 433                     network_dict["provider:network_type"] = vlan
 434
 435                 ####### VIO Specific Changes #########
 436                 if self.vim_type == "VIO":
 437                     if vlan is not None:
 438                         network_dict["provider:segmentation_id"] = vlan
 439                     else:
 440                         if self.config.get('dataplane_net_vlan_range') is None:
 441                             raise vimconn.vimconnConflictException("You must provide "\
 442                                 "'dataplane_net_vlan_range' in format [start_ID - end_ID]"\
 443                                 "at config value before creating sriov network with vlan tag")
 444
 445                         network_dict["provider:segmentation_id"] = self._genrate_vlanID()
 446
 447             network_dict["shared"]=shared
 448             new_net=self.neutron.create_network({'network':network_dict})
 449             #print new_net
 450             #create subnetwork, even if there is no profile
 451             if not ip_profile:
 452                 ip_profile = {}
 453             if not ip_profile.get('subnet_address'):
 454                 #Fake subnet is required
 455                 subnet_rand = random.randint(0, 255)
 456                 ip_profile['subnet_address'] = "192.168.{}.0/24".format(subnet_rand)
 457             if 'ip_version' not in ip_profile:
 458                 ip_profile['ip_version'] = "IPv4"
 459             subnet = {"name":net_name+"-subnet",
 460                     "network_id": new_net["network"]["id"],
 461                     "ip_version": 4 if ip_profile['ip_version']=="IPv4" else 6,
 462                     "cidr": ip_profile['subnet_address']
 463                     }
 464             # Gateway should be set to None if not needed. Otherwise openstack assigns one by default
 465             if ip_profile.get('gateway_address'):
 466                 subnet['gateway_ip'] = ip_profile['gateway_address']
 467             else:
 468                 subnet['gateway_ip'] = None
 469             if ip_profile.get('dns_address'):
 470                 subnet['dns_nameservers'] = ip_profile['dns_address'].split(";")
 471             if 'dhcp_enabled' in ip_profile:
 472                 subnet['enable_dhcp'] = False if \
 473                     ip_profile['dhcp_enabled']=="false" or ip_profile['dhcp_enabled']==False else True
 474             if ip_profile.get('dhcp_start_address'):
 475                 subnet['allocation_pools'] = []
 476                 subnet['allocation_pools'].append(dict())
 477                 subnet['allocation_pools'][0]['start'] = ip_profile['dhcp_start_address']
 478             if ip_profile.get('dhcp_count'):
 479                 #parts = ip_profile['dhcp_start_address'].split('.')
 480                 #ip_int = (int(parts[0]) << 24) + (int(parts[1]) << 16) + (int(parts[2]) << 8) + int(parts[3])
 481                 ip_int = int(netaddr.IPAddress(ip_profile['dhcp_start_address']))
 482                 ip_int += ip_profile['dhcp_count'] - 1
 483                 ip_str = str(netaddr.IPAddress(ip_int))
 484                 subnet['allocation_pools'][0]['end'] = ip_str
 485             #self.logger.debug(">>>>>>>>>>>>>>>>>> Subnet: %s", str(subnet))
 486             self.neutron.create_subnet({"subnet": subnet} )
 487             return new_net["network"]["id"]
 488         except Exception as e:
 489             if new_net:
 490                 self.neutron.delete_network(new_net['network']['id'])
 491             self._format_exception(e)
 492
 493     def get_network_list(self, filter_dict={}):
 494         '''Obtain tenant networks of VIM
 495         Filter_dict can be:
 496             name: network name
 497             id: network uuid
 498             shared: boolean
 499             tenant_id: tenant
 500             admin_state_up: boolean
 501             status: 'ACTIVE'
 502         Returns the network list of dictionaries
 503         '''
 504         self.logger.debug("Getting network from VIM filter: '%s'", str(filter_dict))
 505         try:
 506             self._reload_connection()
 507             filter_dict_os = filter_dict.copy()
 508             if self.api_version3 and "tenant_id" in filter_dict_os:
 509                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')  #T ODO check
 510             net_dict = self.neutron.list_networks(**filter_dict_os)
 511             net_list = net_dict["networks"]
 512             self.__net_os2mano(net_list)
 513             return net_list
 514         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 515             self._format_exception(e)
 516
 517     def get_network(self, net_id):
 518         '''Obtain details of network from VIM
 519         Returns the network information from a network id'''
 520         self.logger.debug(" Getting tenant network %s from VIM", net_id)
 521         filter_dict={"id": net_id}
 522         net_list = self.get_network_list(filter_dict)
 523         if len(net_list)==0:
 524             raise vimconn.vimconnNotFoundException("Network '{}' not found".format(net_id))
 525         elif len(net_list)>1:
 526             raise vimconn.vimconnConflictException("Found more than one network with this criteria")
 527         net = net_list[0]
 528         subnets=[]
 529         for subnet_id in net.get("subnets", () ):
 530             try:
 531                 subnet = self.neutron.show_subnet(subnet_id)
 532             except Exception as e:
 533                 self.logger.error("osconnector.get_network(): Error getting subnet %s %s" % (net_id, str(e)))
 534                 subnet = {"id": subnet_id, "fault": str(e)}
 535             subnets.append(subnet)
 536         net["subnets"] = subnets
 537         net["encapsulation"] = net.get('provider:network_type')
 538         net["segmentation_id"] = net.get('provider:segmentation_id')
 539         return net
 540
 541     def delete_network(self, net_id):
 542         '''Deletes a tenant network from VIM. Returns the old network identifier'''
 543         self.logger.debug("Deleting network '%s' from VIM", net_id)
 544         try:
 545             self._reload_connection()
 546             #delete VM ports attached to this networks before the network
 547             ports = self.neutron.list_ports(network_id=net_id)
 548             for p in ports['ports']:
 549                 try:
 550                     self.neutron.delete_port(p["id"])
 551                 except Exception as e:
 552                     self.logger.error("Error deleting port %s: %s", p["id"], str(e))
 553             self.neutron.delete_network(net_id)
 554             return net_id
 555         except (neExceptions.ConnectionFailed, neExceptions.NetworkNotFoundClient, neExceptions.NeutronException,
 556                 ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 557             self._format_exception(e)
 558
 559     def refresh_nets_status(self, net_list):
 560         '''Get the status of the networks
 561            Params: the list of network identifiers
 562            Returns a dictionary with:
 563                 net_id:         #VIM id of this network
 564                     status:     #Mandatory. Text with one of:
 565                                 #  DELETED (not found at vim)
 566                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 567                                 #  OTHER (Vim reported other status not understood)
 568                                 #  ERROR (VIM indicates an ERROR status)
 569                                 #  ACTIVE, INACTIVE, DOWN (admin down),
 570                                 #  BUILD (on building process)
 571                                 #
 572                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 573                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 574
 575         '''
 576         net_dict={}
 577         for net_id in net_list:
 578             net = {}
 579             try:
 580                 net_vim = self.get_network(net_id)
 581                 if net_vim['status'] in netStatus2manoFormat:
 582                     net["status"] = netStatus2manoFormat[ net_vim['status'] ]
 583                 else:
 584                     net["status"] = "OTHER"
 585                     net["error_msg"] = "VIM status reported " + net_vim['status']
 586
 587                 if net['status'] == "ACTIVE" and not net_vim['admin_state_up']:
 588                     net['status'] = 'DOWN'
 589                 try:
 590                     net['vim_info'] = yaml.safe_dump(net_vim, default_flow_style=True, width=256)
 591                 except yaml.representer.RepresenterError:
 592                     net['vim_info'] = str(net_vim)
 593                 if net_vim.get('fault'):  #TODO
 594                     net['error_msg'] = str(net_vim['fault'])
 595             except vimconn.vimconnNotFoundException as e:
 596                 self.logger.error("Exception getting net status: %s", str(e))
 597                 net['status'] = "DELETED"
 598                 net['error_msg'] = str(e)
 599             except vimconn.vimconnException as e:
 600                 self.logger.error("Exception getting net status: %s", str(e))
 601                 net['status'] = "VIM_ERROR"
 602                 net['error_msg'] = str(e)
 603             net_dict[net_id] = net
 604         return net_dict
 605
 606     def get_flavor(self, flavor_id):
 607         '''Obtain flavor details from the  VIM. Returns the flavor dict details'''
 608         self.logger.debug("Getting flavor '%s'", flavor_id)
 609         try:
 610             self._reload_connection()
 611             flavor = self.nova.flavors.find(id=flavor_id)
 612             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 613             return flavor.to_dict()
 614         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 615             self._format_exception(e)
 616
 617     def get_flavor_id_from_data(self, flavor_dict):
 618         """Obtain flavor id that match the flavor description
 619            Returns the flavor_id or raises a vimconnNotFoundException
 620            flavor_dict: contains the required ram, vcpus, disk
 621            If 'use_existing_flavors' is set to True at config, the closer flavor that provides same or more ram, vcpus
 622                 and disk is returned. Otherwise a flavor with exactly same ram, vcpus and disk is returned or a
 623                 vimconnNotFoundException is raised
 624         """
 625         exact_match = False if self.config.get('use_existing_flavors') else True
 626         try:
 627             self._reload_connection()
 628             flavor_candidate_id = None
 629             flavor_candidate_data = (10000, 10000, 10000)
 630             flavor_target = (flavor_dict["ram"], flavor_dict["vcpus"], flavor_dict["disk"])
 631             # numa=None
 632             numas = flavor_dict.get("extended", {}).get("numas")
 633             if numas:
 634                 #TODO
 635                 raise vimconn.vimconnNotFoundException("Flavor with EPA still not implemted")
 636                 # if len(numas) > 1:
 637                 #     raise vimconn.vimconnNotFoundException("Cannot find any flavor with more than one numa")
 638                 # numa=numas[0]
 639                 # numas = extended.get("numas")
 640             for flavor in self.nova.flavors.list():
 641                 epa = flavor.get_keys()
 642                 if epa:
 643                     continue
 644                     # TODO
 645                 flavor_data = (flavor.ram, flavor.vcpus, flavor.disk)
 646                 if flavor_data == flavor_target:
 647                     return flavor.id
 648                 elif not exact_match and flavor_target < flavor_data < flavor_candidate_data:
 649                     flavor_candidate_id = flavor.id
 650                     flavor_candidate_data = flavor_data
 651             if not exact_match and flavor_candidate_id:
 652                 return flavor_candidate_id
 653             raise vimconn.vimconnNotFoundException("Cannot find any flavor matching '{}'".format(str(flavor_dict)))
 654         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 655             self._format_exception(e)
 656
 657     def new_flavor(self, flavor_data, change_name_if_used=True):
 658         '''Adds a tenant flavor to openstack VIM
 659         if change_name_if_used is True, it will change name in case of conflict, because it is not supported name repetition
 660         Returns the flavor identifier
 661         '''
 662         self.logger.debug("Adding flavor '%s'", str(flavor_data))
 663         retry=0
 664         max_retries=3
 665         name_suffix = 0
 666         try:
 667             name=flavor_data['name']
 668             while retry<max_retries:
 669                 retry+=1
 670                 try:
 671                     self._reload_connection()
 672                     if change_name_if_used:
 673                         #get used names
 674                         fl_names=[]
 675                         fl=self.nova.flavors.list()
 676                         for f in fl:
 677                             fl_names.append(f.name)
 678                         while name in fl_names:
 679                             name_suffix += 1
 680                             name = flavor_data['name']+"-" + str(name_suffix)
 681
 682                     ram = flavor_data.get('ram',64)
 683                     vcpus = flavor_data.get('vcpus',1)
 684                     numa_properties=None
 685
 686                     extended = flavor_data.get("extended")
 687                     if extended:
 688                         numas=extended.get("numas")
 689                         if numas:
 690                             numa_nodes = len(numas)
 691                             if numa_nodes > 1:
 692                                 return -1, "Can not add flavor with more than one numa"
 693                             numa_properties = {"hw:numa_nodes":str(numa_nodes)}
 694                             numa_properties["hw:mem_page_size"] = "large"
 695                             numa_properties["hw:cpu_policy"] = "dedicated"
 696                             numa_properties["hw:numa_mempolicy"] = "strict"
 697                             if self.vim_type == "VIO":
 698                                 numa_properties["vmware:extra_config"] = '{"numa.nodeAffinity":"0"}'
 699                                 numa_properties["vmware:latency_sensitivity_level"] = "high"
 700                             for numa in numas:
 701                                 #overwrite ram and vcpus
 702                                 #check if key 'memory' is present in numa else use ram value at flavor
 703                                 if 'memory' in numa:
 704                                     ram = numa['memory']*1024
 705                                 #See for reference: https://specs.openstack.org/openstack/nova-specs/specs/mitaka/implemented/virt-driver-cpu-thread-pinning.html
 706                                 if 'paired-threads' in numa:
 707                                     vcpus = numa['paired-threads']*2
 708                                     #cpu_thread_policy "require" implies that the compute node must have an STM architecture
 709                                     numa_properties["hw:cpu_thread_policy"] = "require"
 710                                     numa_properties["hw:cpu_policy"] = "dedicated"
 711                                 elif 'cores' in numa:
 712                                     vcpus = numa['cores']
 713                                     # cpu_thread_policy "prefer" implies that the host must not have an SMT architecture, or a non-SMT architecture will be emulated
 714                                     numa_properties["hw:cpu_thread_policy"] = "isolate"
 715                                     numa_properties["hw:cpu_policy"] = "dedicated"
 716                                 elif 'threads' in numa:
 717                                     vcpus = numa['threads']
 718                                     # cpu_thread_policy "prefer" implies that the host may or may not have an SMT architecture
 719                                     numa_properties["hw:cpu_thread_policy"] = "prefer"
 720                                     numa_properties["hw:cpu_policy"] = "dedicated"
 721                                 # for interface in numa.get("interfaces",() ):
 722                                 #     if interface["dedicated"]=="yes":
 723                                 #         raise vimconn.vimconnException("Passthrough interfaces are not supported for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable)
 724                                 #     #TODO, add the key 'pci_passthrough:alias"="<label at config>:<number ifaces>"' when a way to connect it is available
 725
 726                     #create flavor
 727                     new_flavor=self.nova.flavors.create(name,
 728                                     ram,
 729                                     vcpus,
 730                                     flavor_data.get('disk',0),
 731                                     is_public=flavor_data.get('is_public', True)
 732                                 )
 733                     #add metadata
 734                     if numa_properties:
 735                         new_flavor.set_keys(numa_properties)
 736                     return new_flavor.id
 737                 except nvExceptions.Conflict as e:
 738                     if change_name_if_used and retry < max_retries:
 739                         continue
 740                     self._format_exception(e)
 741         #except nvExceptions.BadRequest as e:
 742         except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError, KeyError) as e:
 743             self._format_exception(e)
 744
 745     def delete_flavor(self,flavor_id):
 746         '''Deletes a tenant flavor from openstack VIM. Returns the old flavor_id
 747         '''
 748         try:
 749             self._reload_connection()
 750             self.nova.flavors.delete(flavor_id)
 751             return flavor_id
 752         #except nvExceptions.BadRequest as e:
 753         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 754             self._format_exception(e)
 755
 756     def new_image(self,image_dict):
 757         '''
 758         Adds a tenant image to VIM. imge_dict is a dictionary with:
 759             name: name
 760             disk_format: qcow2, vhd, vmdk, raw (by default), ...
 761             location: path or URI
 762             public: "yes" or "no"
 763             metadata: metadata of the image
 764         Returns the image_id
 765         '''
 766         retry=0
 767         max_retries=3
 768         while retry<max_retries:
 769             retry+=1
 770             try:
 771                 self._reload_connection()
 772                 #determine format  http://docs.openstack.org/developer/glance/formats.html
 773                 if "disk_format" in image_dict:
 774                     disk_format=image_dict["disk_format"]
 775                 else: #autodiscover based on extension
 776                     if image_dict['location'].endswith(".qcow2"):
 777                         disk_format="qcow2"
 778                     elif image_dict['location'].endswith(".vhd"):
 779                         disk_format="vhd"
 780                     elif image_dict['location'].endswith(".vmdk"):
 781                         disk_format="vmdk"
 782                     elif image_dict['location'].endswith(".vdi"):
 783                         disk_format="vdi"
 784                     elif image_dict['location'].endswith(".iso"):
 785                         disk_format="iso"
 786                     elif image_dict['location'].endswith(".aki"):
 787                         disk_format="aki"
 788                     elif image_dict['location'].endswith(".ari"):
 789                         disk_format="ari"
 790                     elif image_dict['location'].endswith(".ami"):
 791                         disk_format="ami"
 792                     else:
 793                         disk_format="raw"
 794                 self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location'])
 795                 if self.vim_type == "VIO":
 796                     container_format = "bare"
 797                     if 'container_format' in image_dict:
 798                         container_format = image_dict['container_format']
 799                     new_image = self.glance.images.create(name=image_dict['name'], container_format=container_format,
 800                                                           disk_format=disk_format)
 801                 else:
 802                     new_image = self.glance.images.create(name=image_dict['name'])
 803                 if image_dict['location'].startswith("http"):
 804                     # TODO there is not a method to direct download. It must be downloaded locally with requests
 805                     raise vimconn.vimconnNotImplemented("Cannot create image from URL")
 806                 else: #local path
 807                     with open(image_dict['location']) as fimage:
 808                         self.glance.images.upload(new_image.id, fimage)
 809                         #new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 810                         #    container_format="bare", data=fimage, disk_format=disk_format)
 811                 metadata_to_load = image_dict.get('metadata')
 812                 # TODO location is a reserved word for current openstack versions. fixed for VIO please check for openstack
 813                 if self.vim_type == "VIO":
 814                     metadata_to_load['upload_location'] = image_dict['location']
 815                 else:
 816                     metadata_to_load['location'] = image_dict['location']
 817                 self.glance.images.update(new_image.id, **metadata_to_load)
 818                 return new_image.id
 819             except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e:
 820                 self._format_exception(e)
 821             except (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 822                 if retry==max_retries:
 823                     continue
 824                 self._format_exception(e)
 825             except IOError as e:  #can not open the file
 826                 raise vimconn.vimconnConnectionException(type(e).__name__ + ": " + str(e)+ " for " + image_dict['location'],
 827                                                          http_code=vimconn.HTTP_Bad_Request)
 828
 829     def delete_image(self, image_id):
 830         '''Deletes a tenant image from openstack VIM. Returns the old id
 831         '''
 832         try:
 833             self._reload_connection()
 834             self.glance.images.delete(image_id)
 835             return image_id
 836         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, gl1Exceptions.HTTPNotFound, ConnectionError) as e: #TODO remove
 837             self._format_exception(e)
 838
 839     def get_image_id_from_path(self, path):
 840         '''Get the image id from image path in the VIM database. Returns the image_id'''
 841         try:
 842             self._reload_connection()
 843             images = self.glance.images.list()
 844             for image in images:
 845                 if image.metadata.get("location")==path:
 846                     return image.id
 847             raise vimconn.vimconnNotFoundException("image with location '{}' not found".format( path))
 848         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 849             self._format_exception(e)
 850
 851     def get_image_list(self, filter_dict={}):
 852         '''Obtain tenant images from VIM
 853         Filter_dict can be:
 854             id: image id
 855             name: image name
 856             checksum: image checksum
 857         Returns the image list of dictionaries:
 858             [{<the fields at Filter_dict plus some VIM specific>}, ...]
 859             List can be empty
 860         '''
 861         self.logger.debug("Getting image list from VIM filter: '%s'", str(filter_dict))
 862         try:
 863             self._reload_connection()
 864             filter_dict_os = filter_dict.copy()
 865             #First we filter by the available filter fields: name, id. The others are removed.
 866             image_list = self.glance.images.list()
 867             filtered_list = []
 868             for image in image_list:
 869                 try:
 870                     if filter_dict.get("name") and image["name"] != filter_dict["name"]:
 871                         continue
 872                     if filter_dict.get("id") and image["id"] != filter_dict["id"]:
 873                         continue
 874                     if filter_dict.get("checksum") and image["checksum"] != filter_dict["checksum"]:
 875                         continue
 876
 877                     filtered_list.append(image.copy())
 878                 except gl1Exceptions.HTTPNotFound:
 879                     pass
 880             return filtered_list
 881         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 882             self._format_exception(e)
 883
 884     def __wait_for_vm(self, vm_id, status):
 885         """wait until vm is in the desired status and return True.
 886         If the VM gets in ERROR status, return false.
 887         If the timeout is reached generate an exception"""
 888         elapsed_time = 0
 889         while elapsed_time < server_timeout:
 890             vm_status = self.nova.servers.get(vm_id).status
 891             if vm_status == status:
 892                 return True
 893             if vm_status == 'ERROR':
 894                 return False
 895             time.sleep(5)
 896             elapsed_time += 5
 897
 898         # if we exceeded the timeout rollback
 899         if elapsed_time >= server_timeout:
 900             raise vimconn.vimconnException('Timeout waiting for instance ' + vm_id + ' to get ' + status,
 901                                            http_code=vimconn.HTTP_Request_Timeout)
 902
 903     def _get_openstack_availablity_zones(self):
 904         """
 905         Get from openstack availability zones available
 906         :return:
 907         """
 908         try:
 909             openstack_availability_zone = self.nova.availability_zones.list()
 910             openstack_availability_zone = [str(zone.zoneName) for zone in openstack_availability_zone
 911                                            if zone.zoneName != 'internal']
 912             return openstack_availability_zone
 913         except Exception as e:
 914             return None
 915
 916     def _set_availablity_zones(self):
 917         """
 918         Set vim availablity zone
 919         :return:
 920         """
 921
 922         if 'availability_zone' in self.config:
 923             vim_availability_zones = self.config.get('availability_zone')
 924             if isinstance(vim_availability_zones, str):
 925                 self.availability_zone = [vim_availability_zones]
 926             elif isinstance(vim_availability_zones, list):
 927                 self.availability_zone = vim_availability_zones
 928         else:
 929             self.availability_zone = self._get_openstack_availablity_zones()
 930
 931     def _get_vm_availability_zone(self, availability_zone_index, availability_zone_list):
 932         """
 933         Return thge availability zone to be used by the created VM.
 934         :return: The VIM availability zone to be used or None
 935         """
 936         if availability_zone_index is None:
 937             if not self.config.get('availability_zone'):
 938                 return None
 939             elif isinstance(self.config.get('availability_zone'), str):
 940                 return self.config['availability_zone']
 941             else:
 942                 # TODO consider using a different parameter at config for default AV and AV list match
 943                 return self.config['availability_zone'][0]
 944
 945         vim_availability_zones = self.availability_zone
 946         # check if VIM offer enough availability zones describe in the VNFD
 947         if vim_availability_zones and len(availability_zone_list) <= len(vim_availability_zones):
 948             # check if all the names of NFV AV match VIM AV names
 949             match_by_index = False
 950             for av in availability_zone_list:
 951                 if av not in vim_availability_zones:
 952                     match_by_index = True
 953                     break
 954             if match_by_index:
 955                 return vim_availability_zones[availability_zone_index]
 956             else:
 957                 return availability_zone_list[availability_zone_index]
 958         else:
 959             raise vimconn.vimconnConflictException("No enough availability zones at VIM for this deployment")
 960
 961     def new_vminstance(self, name, description, start, image_id, flavor_id, net_list, cloud_config=None, disk_list=None,
 962                        availability_zone_index=None, availability_zone_list=None):
 963         """Adds a VM instance to VIM
 964         Params:
 965             start: indicates if VM must start or boot in pause mode. Ignored
 966             image_id,flavor_id: iamge and flavor uuid
 967             net_list: list of interfaces, each one is a dictionary with:
 968                 name:
 969                 net_id: network uuid to connect
 970                 vpci: virtual vcpi to assign, ignored because openstack lack #TODO
 971                 model: interface model, ignored #TODO
 972                 mac_address: used for  SR-IOV ifaces #TODO for other types
 973                 use: 'data', 'bridge',  'mgmt'
 974                 type: 'virtual', 'PCI-PASSTHROUGH'('PF'), 'SR-IOV'('VF'), 'VFnotShared'
 975                 vim_id: filled/added by this function
 976                 floating_ip: True/False (or it can be None)
 977             'cloud_config': (optional) dictionary with:
 978             'key-pairs': (optional) list of strings with the public key to be inserted to the default user
 979             'users': (optional) list of users to be inserted, each item is a dict with:
 980                 'name': (mandatory) user name,
 981                 'key-pairs': (optional) list of strings with the public key to be inserted to the user
 982             'user-data': (optional) string is a text script to be passed directly to cloud-init
 983             'config-files': (optional). List of files to be transferred. Each item is a dict with:
 984                 'dest': (mandatory) string with the destination absolute path
 985                 'encoding': (optional, by default text). Can be one of:
 986                     'b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64'
 987                 'content' (mandatory): string with the content of the file
 988                 'permissions': (optional) string with file permissions, typically octal notation '0644'
 989                 'owner': (optional) file owner, string with the format 'owner:group'
 990             'boot-data-drive': boolean to indicate if user-data must be passed using a boot drive (hard disk)
 991             'disk_list': (optional) list with additional disks to the VM. Each item is a dict with:
 992                 'image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted
 993                 'size': (mandatory) string with the size of the disk in GB
 994                 'vim_id' (optional) should use this existing volume id
 995             availability_zone_index: Index of availability_zone_list to use for this this VM. None if not AV required
 996             availability_zone_list: list of availability zones given by user in the VNFD descriptor.  Ignore if
 997                 availability_zone_index is None
 998                 #TODO ip, security groups
 999         Returns a tuple with the instance identifier and created_items or raises an exception on error
1000             created_items can be None or a dictionary where this method can include key-values that will be passed to
1001             the method delete_vminstance and action_vminstance. Can be used to store created ports, volumes, etc.
1002             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
1003             as not present.
1004         """
1005         self.logger.debug("new_vminstance input: image='%s' flavor='%s' nics='%s'",image_id, flavor_id,str(net_list))
1006         try:
1007             server = None
1008             created_items = {}
1009             # metadata = {}
1010             net_list_vim = []
1011             external_network = []   # list of external networks to be connected to instance, later on used to create floating_ip
1012             no_secured_ports = []   # List of port-is with port-security disabled
1013             self._reload_connection()
1014             # metadata_vpci = {}   # For a specific neutron plugin
1015             block_device_mapping = None
1016             for net in net_list:
1017                 if not net.get("net_id"):   # skip non connected iface
1018                     continue
1019
1020                 port_dict={
1021                     "network_id": net["net_id"],
1022                     "name": net.get("name"),
1023                     "admin_state_up": True
1024                 }
1025                 if net["type"]=="virtual":
1026                     pass
1027                     # if "vpci" in net:
1028                     #     metadata_vpci[ net["net_id"] ] = [[ net["vpci"], "" ]]
1029                 elif net["type"] == "VF" or net["type"] == "SR-IOV":  # for VF
1030                     # if "vpci" in net:
1031                     #     if "VF" not in metadata_vpci:
1032                     #         metadata_vpci["VF"]=[]
1033                     #     metadata_vpci["VF"].append([ net["vpci"], "" ])
1034                     port_dict["binding:vnic_type"]="direct"
1035                     # VIO specific Changes
1036                     if self.vim_type == "VIO":
1037                         # Need to create port with port_security_enabled = False and no-security-groups
1038                         port_dict["port_security_enabled"]=False
1039                         port_dict["provider_security_groups"]=[]
1040                         port_dict["security_groups"]=[]
1041                 else:   # For PT PCI-PASSTHROUGH
1042                     # VIO specific Changes
1043                     # Current VIO release does not support port with type 'direct-physical'
1044                     # So no need to create virtual port in case of PCI-device.
1045                     # Will update port_dict code when support gets added in next VIO release
1046                     if self.vim_type == "VIO":
1047                         raise vimconn.vimconnNotSupportedException(
1048                             "Current VIO release does not support full passthrough (PT)")
1049                     # if "vpci" in net:
1050                     #     if "PF" not in metadata_vpci:
1051                     #         metadata_vpci["PF"]=[]
1052                     #     metadata_vpci["PF"].append([ net["vpci"], "" ])
1053                     port_dict["binding:vnic_type"]="direct-physical"
1054                 if not port_dict["name"]:
1055                     port_dict["name"]=name
1056                 if net.get("mac_address"):
1057                     port_dict["mac_address"]=net["mac_address"]
1058                 if net.get("ip_address"):
1059                     port_dict["fixed_ips"] = [{'ip_address': net["ip_address"]}]
1060                     # TODO add 'subnet_id': <subnet_id>
1061                 new_port = self.neutron.create_port({"port": port_dict })
1062                 created_items["port:" + str(new_port["port"]["id"])] = True
1063                 net["mac_adress"] = new_port["port"]["mac_address"]
1064                 net["vim_id"] = new_port["port"]["id"]
1065                 # if try to use a network without subnetwork, it will return a emtpy list
1066                 fixed_ips = new_port["port"].get("fixed_ips")
1067                 if fixed_ips:
1068                     net["ip"] = fixed_ips[0].get("ip_address")
1069                 else:
1070                     net["ip"] = None
1071
1072                 port = {"port-id": new_port["port"]["id"]}
1073                 if float(self.nova.api_version.get_string()) >= 2.32:
1074                     port["tag"] = new_port["port"]["name"]
1075                 net_list_vim.append(port)
1076
1077                 if net.get('floating_ip', False):
1078                     net['exit_on_floating_ip_error'] = True
1079                     external_network.append(net)
1080                 elif net['use'] == 'mgmt' and self.config.get('use_floating_ip'):
1081                     net['exit_on_floating_ip_error'] = False
1082                     external_network.append(net)
1083                     net['floating_ip'] = self.config.get('use_floating_ip')
1084
1085                 # If port security is disabled when the port has not yet been attached to the VM, then all vm traffic is dropped.
1086                 # As a workaround we wait until the VM is active and then disable the port-security
1087                 if net.get("port_security") == False and not self.config.get("no_port_security_extension"):
1088                     no_secured_ports.append(new_port["port"]["id"])
1089
1090             # if metadata_vpci:
1091             #     metadata = {"pci_assignement": json.dumps(metadata_vpci)}
1092             #     if len(metadata["pci_assignement"]) >255:
1093             #         #limit the metadata size
1094             #         #metadata["pci_assignement"] = metadata["pci_assignement"][0:255]
1095             #         self.logger.warn("Metadata deleted since it exceeds the expected length (255) ")
1096             #         metadata = {}
1097
1098             self.logger.debug("name '%s' image_id '%s'flavor_id '%s' net_list_vim '%s' description '%s'",
1099                               name, image_id, flavor_id, str(net_list_vim), description)
1100
1101             security_groups = self.config.get('security_groups')
1102             if type(security_groups) is str:
1103                 security_groups = ( security_groups, )
1104             # cloud config
1105             config_drive, userdata = self._create_user_data(cloud_config)
1106
1107             # Create additional volumes in case these are present in disk_list
1108             base_disk_index = ord('b')
1109             if disk_list:
1110                 block_device_mapping = {}
1111                 for disk in disk_list:
1112                     if disk.get('vim_id'):
1113                         block_device_mapping['_vd' + chr(base_disk_index)] = disk['vim_id']
1114                     else:
1115                         if 'image_id' in disk:
1116                             volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1117                                                                 chr(base_disk_index), imageRef=disk['image_id'])
1118                         else:
1119                             volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1120                                                                 chr(base_disk_index))
1121                         created_items["volume:" + str(volume.id)] = True
1122                         block_device_mapping['_vd' + chr(base_disk_index)] = volume.id
1123                     base_disk_index += 1
1124
1125                 # Wait until created volumes are with status available
1126                 elapsed_time = 0
1127                 while elapsed_time < volume_timeout:
1128                     for created_item in created_items:
1129                         v, _, volume_id = created_item.partition(":")
1130                         if v == 'volume':
1131                             if self.cinder.volumes.get(volume_id).status != 'available':
1132                                 break
1133                     else:  # all ready: break from while
1134                         break
1135                     time.sleep(5)
1136                     elapsed_time += 5
1137                 # If we exceeded the timeout rollback
1138                 if elapsed_time >= volume_timeout:
1139                     raise vimconn.vimconnException('Timeout creating volumes for instance ' + name,
1140                                                    http_code=vimconn.HTTP_Request_Timeout)
1141             # get availability Zone
1142             vm_av_zone = self._get_vm_availability_zone(availability_zone_index, availability_zone_list)
1143
1144             self.logger.debug("nova.servers.create({}, {}, {}, nics={}, security_groups={}, "
1145                               "availability_zone={}, key_name={}, userdata={}, config_drive={}, "
1146                               "block_device_mapping={})".format(name, image_id, flavor_id, net_list_vim,
1147                                                                 security_groups, vm_av_zone, self.config.get('keypair'),
1148                                                                 userdata, config_drive, block_device_mapping))
1149             server = self.nova.servers.create(name, image_id, flavor_id, nics=net_list_vim,
1150                                               security_groups=security_groups,
1151                                               availability_zone=vm_av_zone,
1152                                               key_name=self.config.get('keypair'),
1153                                               userdata=userdata,
1154                                               config_drive=config_drive,
1155                                               block_device_mapping=block_device_mapping
1156                                               )  # , description=description)
1157
1158             vm_start_time = time.time()
1159             # Previously mentioned workaround to wait until the VM is active and then disable the port-security
1160             if no_secured_ports:
1161                 self.__wait_for_vm(server.id, 'ACTIVE')
1162
1163             for port_id in no_secured_ports:
1164                 try:
1165                     self.neutron.update_port(port_id,
1166                                              {"port": {"port_security_enabled": False, "security_groups": None}})
1167                 except Exception as e:
1168                     raise vimconn.vimconnException("It was not possible to disable port security for port {}".format(
1169                         port_id))
1170             # print "DONE :-)", server
1171
1172             # pool_id = None
1173             if external_network:
1174                 floating_ips = self.neutron.list_floatingips().get("floatingips", ())
1175             for floating_network in external_network:
1176                 try:
1177                     assigned = False
1178                     while not assigned:
1179                         if floating_ips:
1180                             ip = floating_ips.pop(0)
1181                             if ip.get("port_id", False) or ip.get('tenant_id') != server.tenant_id:
1182                                 continue
1183                             if isinstance(floating_network['floating_ip'], str):
1184                                 if ip.get("floating_network_id") != floating_network['floating_ip']:
1185                                     continue
1186                             free_floating_ip = ip.get("floating_ip_address")
1187                         else:
1188                             if isinstance(floating_network['floating_ip'], str) and \
1189                                 floating_network['floating_ip'].lower() != "true":
1190                                 pool_id = floating_network['floating_ip']
1191                             else:
1192                                 # Find the external network
1193                                 external_nets = list()
1194                                 for net in self.neutron.list_networks()['networks']:
1195                                     if net['router:external']:
1196                                             external_nets.append(net)
1197
1198                                 if len(external_nets) == 0:
1199                                     raise vimconn.vimconnException("Cannot create floating_ip automatically since no external "
1200                                                                    "network is present",
1201                                                                     http_code=vimconn.HTTP_Conflict)
1202                                 if len(external_nets) > 1:
1203                                     raise vimconn.vimconnException("Cannot create floating_ip automatically since multiple "
1204                                                                    "external networks are present",
1205                                                                    http_code=vimconn.HTTP_Conflict)
1206
1207                                 pool_id = external_nets[0].get('id')
1208                             param = {'floatingip': {'floating_network_id': pool_id, 'tenant_id': server.tenant_id}}
1209                             try:
1210                                 # self.logger.debug("Creating floating IP")
1211                                 new_floating_ip = self.neutron.create_floatingip(param)
1212                                 free_floating_ip = new_floating_ip['floatingip']['floating_ip_address']
1213                             except Exception as e:
1214                                 raise vimconn.vimconnException(type(e).__name__ + ": Cannot create new floating_ip " +
1215                                                                str(e), http_code=vimconn.HTTP_Conflict)
1216
1217                         fix_ip = floating_network.get('ip')
1218                         while not assigned:
1219                             try:
1220                                 server.add_floating_ip(free_floating_ip, fix_ip)
1221                                 assigned = True
1222                             except Exception as e:
1223                                 # openstack need some time after VM creation to asign an IP. So retry if fails
1224                                 vm_status = self.nova.servers.get(server.id).status
1225                                 if vm_status != 'ACTIVE' and vm_status != 'ERROR':
1226                                     if time.time() - vm_start_time < server_timeout:
1227                                         time.sleep(5)
1228                                         continue
1229                                 raise vimconn.vimconnException(
1230                                     "Cannot create floating_ip: {} {}".format(type(e).__name__, e),
1231                                     http_code=vimconn.HTTP_Conflict)
1232
1233                 except Exception as e:
1234                     if not floating_network['exit_on_floating_ip_error']:
1235                         self.logger.warn("Cannot create floating_ip. %s", str(e))
1236                         continue
1237                     raise
1238
1239             return server.id, created_items
1240 #        except nvExceptions.NotFound as e:
1241 #            error_value=-vimconn.HTTP_Not_Found
1242 #            error_text= "vm instance %s not found" % vm_id
1243 #        except TypeError as e:
1244 #            raise vimconn.vimconnException(type(e).__name__ + ": "+  str(e), http_code=vimconn.HTTP_Bad_Request)
1245
1246         except Exception as e:
1247             server_id = None
1248             if server:
1249                 server_id = server.id
1250             try:
1251                 self.delete_vminstance(server_id, created_items)
1252             except Exception as e2:
1253                 self.logger.error("new_vminstance rollback fail {}".format(e2))
1254
1255             self._format_exception(e)
1256
1257     def get_vminstance(self,vm_id):
1258         '''Returns the VM instance information from VIM'''
1259         #self.logger.debug("Getting VM from VIM")
1260         try:
1261             self._reload_connection()
1262             server = self.nova.servers.find(id=vm_id)
1263             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
1264             return server.to_dict()
1265         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1266             self._format_exception(e)
1267
1268     def get_vminstance_console(self,vm_id, console_type="vnc"):
1269         '''
1270         Get a console for the virtual machine
1271         Params:
1272             vm_id: uuid of the VM
1273             console_type, can be:
1274                 "novnc" (by default), "xvpvnc" for VNC types,
1275                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
1276         Returns dict with the console parameters:
1277                 protocol: ssh, ftp, http, https, ...
1278                 server:   usually ip address
1279                 port:     the http, ssh, ... port
1280                 suffix:   extra text, e.g. the http path and query string
1281         '''
1282         self.logger.debug("Getting VM CONSOLE from VIM")
1283         try:
1284             self._reload_connection()
1285             server = self.nova.servers.find(id=vm_id)
1286             if console_type == None or console_type == "novnc":
1287                 console_dict = server.get_vnc_console("novnc")
1288             elif console_type == "xvpvnc":
1289                 console_dict = server.get_vnc_console(console_type)
1290             elif console_type == "rdp-html5":
1291                 console_dict = server.get_rdp_console(console_type)
1292             elif console_type == "spice-html5":
1293                 console_dict = server.get_spice_console(console_type)
1294             else:
1295                 raise vimconn.vimconnException("console type '{}' not allowed".format(console_type), http_code=vimconn.HTTP_Bad_Request)
1296
1297             console_dict1 = console_dict.get("console")
1298             if console_dict1:
1299                 console_url = console_dict1.get("url")
1300                 if console_url:
1301                     #parse console_url
1302                     protocol_index = console_url.find("//")
1303                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1304                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1305                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1306                         return -vimconn.HTTP_Internal_Server_Error, "Unexpected response from VIM"
1307                     console_dict={"protocol": console_url[0:protocol_index],
1308                                   "server":   console_url[protocol_index+2:port_index],
1309                                   "port":     console_url[port_index:suffix_index],
1310                                   "suffix":   console_url[suffix_index+1:]
1311                                   }
1312                     protocol_index += 2
1313                     return console_dict
1314             raise vimconn.vimconnUnexpectedResponse("Unexpected response from VIM")
1315
1316         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.BadRequest, ConnectionError) as e:
1317             self._format_exception(e)
1318
1319     def delete_vminstance(self, vm_id, created_items=None):
1320         '''Removes a VM instance from VIM. Returns the old identifier
1321         '''
1322         #print "osconnector: Getting VM from VIM"
1323         if created_items == None:
1324             created_items = {}
1325         try:
1326             self._reload_connection()
1327             # delete VM ports attached to this networks before the virtual machine
1328             for k, v in created_items.items():
1329                 if not v:  # skip already deleted
1330                     continue
1331                 try:
1332                     k_item, _, k_id = k.partition(":")
1333                     if k_item == "port":
1334                         self.neutron.delete_port(k_id)
1335                 except Exception as e:
1336                     self.logger.error("Error deleting port: {}: {}".format(type(e).__name__, e))
1337
1338             # #commented because detaching the volumes makes the servers.delete not work properly ?!?
1339             # #dettach volumes attached
1340             # server = self.nova.servers.get(vm_id)
1341             # volumes_attached_dict = server._info['os-extended-volumes:volumes_attached']   #volume['id']
1342             # #for volume in volumes_attached_dict:
1343             # #    self.cinder.volumes.detach(volume['id'])
1344
1345             if vm_id:
1346                 self.nova.servers.delete(vm_id)
1347
1348             # delete volumes. Although having detached, they should have in active status before deleting
1349             # we ensure in this loop
1350             keep_waiting = True
1351             elapsed_time = 0
1352             while keep_waiting and elapsed_time < volume_timeout:
1353                 keep_waiting = False
1354                 for k, v in created_items.items():
1355                     if not v:  # skip already deleted
1356                         continue
1357                     try:
1358                         k_item, _, k_id = k.partition(":")
1359                         if k_item == "volume":
1360                             if self.cinder.volumes.get(k_id).status != 'available':
1361                                 keep_waiting = True
1362                             else:
1363                                 self.cinder.volumes.delete(k_id)
1364                     except Exception as e:
1365                         self.logger.error("Error deleting volume: {}: {}".format(type(e).__name__, e))
1366                 if keep_waiting:
1367                     time.sleep(1)
1368                     elapsed_time += 1
1369             return None
1370         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
1371             self._format_exception(e)
1372
1373     def refresh_vms_status(self, vm_list):
1374         '''Get the status of the virtual machines and their interfaces/ports
1375            Params: the list of VM identifiers
1376            Returns a dictionary with:
1377                 vm_id:          #VIM id of this Virtual Machine
1378                     status:     #Mandatory. Text with one of:
1379                                 #  DELETED (not found at vim)
1380                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1381                                 #  OTHER (Vim reported other status not understood)
1382                                 #  ERROR (VIM indicates an ERROR status)
1383                                 #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
1384                                 #  CREATING (on building process), ERROR
1385                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
1386                                 #
1387                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
1388                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
1389                     interfaces:
1390                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
1391                         mac_address:      #Text format XX:XX:XX:XX:XX:XX
1392                         vim_net_id:       #network id where this interface is connected
1393                         vim_interface_id: #interface/port VIM id
1394                         ip_address:       #null, or text with IPv4, IPv6 address
1395                         compute_node:     #identification of compute node where PF,VF interface is allocated
1396                         pci:              #PCI address of the NIC that hosts the PF,VF
1397                         vlan:             #physical VLAN used for VF
1398         '''
1399         vm_dict={}
1400         self.logger.debug("refresh_vms status: Getting tenant VM instance information from VIM")
1401         for vm_id in vm_list:
1402             vm={}
1403             try:
1404                 vm_vim = self.get_vminstance(vm_id)
1405                 if vm_vim['status'] in vmStatus2manoFormat:
1406                     vm['status']    =  vmStatus2manoFormat[ vm_vim['status'] ]
1407                 else:
1408                     vm['status']    = "OTHER"
1409                     vm['error_msg'] = "VIM status reported " + vm_vim['status']
1410                 try:
1411                     vm['vim_info']  = yaml.safe_dump(vm_vim, default_flow_style=True, width=256)
1412                 except yaml.representer.RepresenterError:
1413                     vm['vim_info'] = str(vm_vim)
1414                 vm["interfaces"] = []
1415                 if vm_vim.get('fault'):
1416                     vm['error_msg'] = str(vm_vim['fault'])
1417                 #get interfaces
1418                 try:
1419                     self._reload_connection()
1420                     port_dict = self.neutron.list_ports(device_id=vm_id)
1421                     for port in port_dict["ports"]:
1422                         interface={}
1423                         try:
1424                             interface['vim_info'] = yaml.safe_dump(port, default_flow_style=True, width=256)
1425                         except yaml.representer.RepresenterError:
1426                             interface['vim_info'] = str(port)
1427                         interface["mac_address"] = port.get("mac_address")
1428                         interface["vim_net_id"] = port["network_id"]
1429                         interface["vim_interface_id"] = port["id"]
1430                         # check if OS-EXT-SRV-ATTR:host is there,
1431                         # in case of non-admin credentials, it will be missing
1432                         if vm_vim.get('OS-EXT-SRV-ATTR:host'):
1433                             interface["compute_node"] = vm_vim['OS-EXT-SRV-ATTR:host']
1434                         interface["pci"] = None
1435
1436                         # check if binding:profile is there,
1437                         # in case of non-admin credentials, it will be missing
1438                         if port.get('binding:profile'):
1439                             if port['binding:profile'].get('pci_slot'):
1440                                 # TODO: At the moment sr-iov pci addresses are converted to PF pci addresses by setting the slot to 0x00
1441                                 # TODO: This is just a workaround valid for niantinc. Find a better way to do so
1442                                 #   CHANGE DDDD:BB:SS.F to DDDD:BB:00.(F%2)   assuming there are 2 ports per nic
1443                                 pci = port['binding:profile']['pci_slot']
1444                                 # interface["pci"] = pci[:-4] + "00." + str(int(pci[-1]) % 2)
1445                                 interface["pci"] = pci
1446                         interface["vlan"] = None
1447                         #if network is of type vlan and port is of type direct (sr-iov) then set vlan id
1448                         network = self.neutron.show_network(port["network_id"])
1449                         if network['network'].get('provider:network_type') == 'vlan' and \
1450                             port.get("binding:vnic_type") == "direct":
1451                             interface["vlan"] = network['network'].get('provider:segmentation_id')
1452                         ips=[]
1453                         #look for floating ip address
1454                         try:
1455                             floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
1456                             if floating_ip_dict.get("floatingips"):
1457                                 ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address") )
1458                         except Exception:
1459                             pass
1460
1461                         for subnet in port["fixed_ips"]:
1462                             ips.append(subnet["ip_address"])
1463                         interface["ip_address"] = ";".join(ips)
1464                         vm["interfaces"].append(interface)
1465                 except Exception as e:
1466                     self.logger.error("Error getting vm interface information {}: {}".format(type(e).__name__, e),
1467                                       exc_info=True)
1468             except vimconn.vimconnNotFoundException as e:
1469                 self.logger.error("Exception getting vm status: %s", str(e))
1470                 vm['status'] = "DELETED"
1471                 vm['error_msg'] = str(e)
1472             except vimconn.vimconnException as e:
1473                 self.logger.error("Exception getting vm status: %s", str(e))
1474                 vm['status'] = "VIM_ERROR"
1475                 vm['error_msg'] = str(e)
1476             vm_dict[vm_id] = vm
1477         return vm_dict
1478
1479     def action_vminstance(self, vm_id, action_dict, created_items={}):
1480         '''Send and action over a VM instance from VIM
1481         Returns None or the console dict if the action was successfully sent to the VIM'''
1482         self.logger.debug("Action over VM '%s': %s", vm_id, str(action_dict))
1483         try:
1484             self._reload_connection()
1485             server = self.nova.servers.find(id=vm_id)
1486             if "start" in action_dict:
1487                 if action_dict["start"]=="rebuild":
1488                     server.rebuild()
1489                 else:
1490                     if server.status=="PAUSED":
1491                         server.unpause()
1492                     elif server.status=="SUSPENDED":
1493                         server.resume()
1494                     elif server.status=="SHUTOFF":
1495                         server.start()
1496             elif "pause" in action_dict:
1497                 server.pause()
1498             elif "resume" in action_dict:
1499                 server.resume()
1500             elif "shutoff" in action_dict or "shutdown" in action_dict:
1501                 server.stop()
1502             elif "forceOff" in action_dict:
1503                 server.stop() #TODO
1504             elif "terminate" in action_dict:
1505                 server.delete()
1506             elif "createImage" in action_dict:
1507                 server.create_image()
1508                 #"path":path_schema,
1509                 #"description":description_schema,
1510                 #"name":name_schema,
1511                 #"metadata":metadata_schema,
1512                 #"imageRef": id_schema,
1513                 #"disk": {"oneOf":[{"type": "null"}, {"type":"string"}] },
1514             elif "rebuild" in action_dict:
1515                 server.rebuild(server.image['id'])
1516             elif "reboot" in action_dict:
1517                 server.reboot() #reboot_type='SOFT'
1518             elif "console" in action_dict:
1519                 console_type = action_dict["console"]
1520                 if console_type == None or console_type == "novnc":
1521                     console_dict = server.get_vnc_console("novnc")
1522                 elif console_type == "xvpvnc":
1523                     console_dict = server.get_vnc_console(console_type)
1524                 elif console_type == "rdp-html5":
1525                     console_dict = server.get_rdp_console(console_type)
1526                 elif console_type == "spice-html5":
1527                     console_dict = server.get_spice_console(console_type)
1528                 else:
1529                     raise vimconn.vimconnException("console type '{}' not allowed".format(console_type),
1530                                                    http_code=vimconn.HTTP_Bad_Request)
1531                 try:
1532                     console_url = console_dict["console"]["url"]
1533                     #parse console_url
1534                     protocol_index = console_url.find("//")
1535                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1536                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1537                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1538                         raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1539                     console_dict2={"protocol": console_url[0:protocol_index],
1540                                   "server":   console_url[protocol_index+2 : port_index],
1541                                   "port":     int(console_url[port_index+1 : suffix_index]),
1542                                   "suffix":   console_url[suffix_index+1:]
1543                                   }
1544                     return console_dict2
1545                 except Exception as e:
1546                     raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1547
1548             return None
1549         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1550             self._format_exception(e)
1551         #TODO insert exception vimconn.HTTP_Unauthorized
1552
1553     ####### VIO Specific Changes #########
1554     def _genrate_vlanID(self):
1555         """
1556          Method to get unused vlanID
1557             Args:
1558                 None
1559             Returns:
1560                 vlanID
1561         """
1562         #Get used VLAN IDs
1563         usedVlanIDs = []
1564         networks = self.get_network_list()
1565         for net in networks:
1566             if net.get('provider:segmentation_id'):
1567                 usedVlanIDs.append(net.get('provider:segmentation_id'))
1568         used_vlanIDs = set(usedVlanIDs)
1569
1570         #find unused VLAN ID
1571         for vlanID_range in self.config.get('dataplane_net_vlan_range'):
1572             try:
1573                 start_vlanid , end_vlanid = map(int, vlanID_range.replace(" ", "").split("-"))
1574                 for vlanID in xrange(start_vlanid, end_vlanid + 1):
1575                     if vlanID not in used_vlanIDs:
1576                         return vlanID
1577             except Exception as exp:
1578                 raise vimconn.vimconnException("Exception {} occurred while generating VLAN ID.".format(exp))
1579         else:
1580             raise vimconn.vimconnConflictException("Unable to create the SRIOV VLAN network."\
1581                 " All given Vlan IDs {} are in use.".format(self.config.get('dataplane_net_vlan_range')))
1582
1583
1584     def _validate_vlan_ranges(self, dataplane_net_vlan_range):
1585         """
1586         Method to validate user given vlanID ranges
1587             Args:  None
1588             Returns: None
1589         """
1590         for vlanID_range in dataplane_net_vlan_range:
1591             vlan_range = vlanID_range.replace(" ", "")
1592             #validate format
1593             vlanID_pattern = r'(\d)*-(\d)*$'
1594             match_obj = re.match(vlanID_pattern, vlan_range)
1595             if not match_obj:
1596                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}.You must provide "\
1597                 "'dataplane_net_vlan_range' in format [start_ID - end_ID].".format(vlanID_range))
1598
1599             start_vlanid , end_vlanid = map(int,vlan_range.split("-"))
1600             if start_vlanid <= 0 :
1601                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1602                 "Start ID can not be zero. For VLAN "\
1603                 "networks valid IDs are 1 to 4094 ".format(vlanID_range))
1604             if end_vlanid > 4094 :
1605                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1606                 "End VLAN ID can not be greater than 4094. For VLAN "\
1607                 "networks valid IDs are 1 to 4094 ".format(vlanID_range))
1608
1609             if start_vlanid > end_vlanid:
1610                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1611                     "You must provide a 'dataplane_net_vlan_range' in format start_ID - end_ID and "\
1612                     "start_ID < end_ID ".format(vlanID_range))
1613
1614 #NOT USED FUNCTIONS
1615
1616     def new_external_port(self, port_data):
1617         #TODO openstack if needed
1618         '''Adds a external port to VIM'''
1619         '''Returns the port identifier'''
1620         return -vimconn.HTTP_Internal_Server_Error, "osconnector.new_external_port() not implemented"
1621
1622     def connect_port_network(self, port_id, network_id, admin=False):
1623         #TODO openstack if needed
1624         '''Connects a external port to a network'''
1625         '''Returns status code of the VIM response'''
1626         return -vimconn.HTTP_Internal_Server_Error, "osconnector.connect_port_network() not implemented"
1627
1628     def new_user(self, user_name, user_passwd, tenant_id=None):
1629         '''Adds a new user to openstack VIM'''
1630         '''Returns the user identifier'''
1631         self.logger.debug("osconnector: Adding a new user to VIM")
1632         try:
1633             self._reload_connection()
1634             user=self.keystone.users.create(user_name, password=user_passwd, default_project=tenant_id)
1635             #self.keystone.tenants.add_user(self.k_creds["username"], #role)
1636             return user.id
1637         except ksExceptions.ConnectionError as e:
1638             error_value=-vimconn.HTTP_Bad_Request
1639             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1640         except ksExceptions.ClientException as e: #TODO remove
1641             error_value=-vimconn.HTTP_Bad_Request
1642             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1643         #TODO insert exception vimconn.HTTP_Unauthorized
1644         #if reaching here is because an exception
1645         self.logger.debug("new_user " + error_text)
1646         return error_value, error_text
1647
1648     def delete_user(self, user_id):
1649         '''Delete a user from openstack VIM'''
1650         '''Returns the user identifier'''
1651         if self.debug:
1652             print("osconnector: Deleting  a  user from VIM")
1653         try:
1654             self._reload_connection()
1655             self.keystone.users.delete(user_id)
1656             return 1, user_id
1657         except ksExceptions.ConnectionError as e:
1658             error_value=-vimconn.HTTP_Bad_Request
1659             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1660         except ksExceptions.NotFound as e:
1661             error_value=-vimconn.HTTP_Not_Found
1662             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1663         except ksExceptions.ClientException as e: #TODO remove
1664             error_value=-vimconn.HTTP_Bad_Request
1665             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1666         #TODO insert exception vimconn.HTTP_Unauthorized
1667         #if reaching here is because an exception
1668             self.logger.debug("delete_tenant " + error_text)
1669         return error_value, error_text
1670
1671     def get_hosts_info(self):
1672         '''Get the information of deployed hosts
1673         Returns the hosts content'''
1674         if self.debug:
1675             print("osconnector: Getting Host info from VIM")
1676         try:
1677             h_list=[]
1678             self._reload_connection()
1679             hypervisors = self.nova.hypervisors.list()
1680             for hype in hypervisors:
1681                 h_list.append( hype.to_dict() )
1682             return 1, {"hosts":h_list}
1683         except nvExceptions.NotFound as e:
1684             error_value=-vimconn.HTTP_Not_Found
1685             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1686         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1687             error_value=-vimconn.HTTP_Bad_Request
1688             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1689         #TODO insert exception vimconn.HTTP_Unauthorized
1690         #if reaching here is because an exception
1691         self.logger.debug("get_hosts_info " + error_text)
1692         return error_value, error_text
1693
1694     def get_hosts(self, vim_tenant):
1695         '''Get the hosts and deployed instances
1696         Returns the hosts content'''
1697         r, hype_dict = self.get_hosts_info()
1698         if r<0:
1699             return r, hype_dict
1700         hypervisors = hype_dict["hosts"]
1701         try:
1702             servers = self.nova.servers.list()
1703             for hype in hypervisors:
1704                 for server in servers:
1705                     if server.to_dict()['OS-EXT-SRV-ATTR:hypervisor_hostname']==hype['hypervisor_hostname']:
1706                         if 'vm' in hype:
1707                             hype['vm'].append(server.id)
1708                         else:
1709                             hype['vm'] = [server.id]
1710             return 1, hype_dict
1711         except nvExceptions.NotFound as e:
1712             error_value=-vimconn.HTTP_Not_Found
1713             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1714         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1715             error_value=-vimconn.HTTP_Bad_Request
1716             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1717         #TODO insert exception vimconn.HTTP_Unauthorized
1718         #if reaching here is because an exception
1719         self.logger.debug("get_hosts " + error_text)
1720         return error_value, error_text
1721
1722     def new_classification(self, name, ctype, definition):
1723         self.logger.debug(
1724             'Adding a new (Traffic) Classification to VIM, named %s', name)
1725         try:
1726             new_class = None
1727             self._reload_connection()
1728             if ctype not in supportedClassificationTypes:
1729                 raise vimconn.vimconnNotSupportedException(
1730                         'OpenStack VIM connector doesn\'t support provided '
1731                         'Classification Type {}, supported ones are: '
1732                         '{}'.format(ctype, supportedClassificationTypes))
1733             if not self._validate_classification(ctype, definition):
1734                 raise vimconn.vimconnException(
1735                     'Incorrect Classification definition '
1736                     'for the type specified.')
1737             classification_dict = definition
1738             classification_dict['name'] = name
1739
1740             new_class = self.neutron.create_sfc_flow_classifier(
1741                 {'flow_classifier': classification_dict})
1742             return new_class['flow_classifier']['id']
1743         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1744                 neExceptions.NeutronException, ConnectionError) as e:
1745             self.logger.error(
1746                 'Creation of Classification failed.')
1747             self._format_exception(e)
1748
1749     def get_classification(self, class_id):
1750         self.logger.debug(" Getting Classification %s from VIM", class_id)
1751         filter_dict = {"id": class_id}
1752         class_list = self.get_classification_list(filter_dict)
1753         if len(class_list) == 0:
1754             raise vimconn.vimconnNotFoundException(
1755                 "Classification '{}' not found".format(class_id))
1756         elif len(class_list) > 1:
1757             raise vimconn.vimconnConflictException(
1758                 "Found more than one Classification with this criteria")
1759         classification = class_list[0]
1760         return classification
1761
1762     def get_classification_list(self, filter_dict={}):
1763         self.logger.debug("Getting Classifications from VIM filter: '%s'",
1764                           str(filter_dict))
1765         try:
1766             filter_dict_os = filter_dict.copy()
1767             self._reload_connection()
1768             if self.api_version3 and "tenant_id" in filter_dict_os:
1769                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1770             classification_dict = self.neutron.list_sfc_flow_classifiers(
1771                 **filter_dict_os)
1772             classification_list = classification_dict["flow_classifiers"]
1773             self.__classification_os2mano(classification_list)
1774             return classification_list
1775         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1776                 neExceptions.NeutronException, ConnectionError) as e:
1777             self._format_exception(e)
1778
1779     def delete_classification(self, class_id):
1780         self.logger.debug("Deleting Classification '%s' from VIM", class_id)
1781         try:
1782             self._reload_connection()
1783             self.neutron.delete_sfc_flow_classifier(class_id)
1784             return class_id
1785         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1786                 ksExceptions.ClientException, neExceptions.NeutronException,
1787                 ConnectionError) as e:
1788             self._format_exception(e)
1789
1790     def new_sfi(self, name, ingress_ports, egress_ports, sfc_encap=True):
1791         self.logger.debug(
1792             "Adding a new Service Function Instance to VIM, named '%s'", name)
1793         try:
1794             new_sfi = None
1795             self._reload_connection()
1796             correlation = None
1797             if sfc_encap:
1798                 correlation = 'nsh'
1799             if len(ingress_ports) != 1:
1800                 raise vimconn.vimconnNotSupportedException(
1801                     "OpenStack VIM connector can only have "
1802                     "1 ingress port per SFI")
1803             if len(egress_ports) != 1:
1804                 raise vimconn.vimconnNotSupportedException(
1805                     "OpenStack VIM connector can only have "
1806                     "1 egress port per SFI")
1807             sfi_dict = {'name': name,
1808                         'ingress': ingress_ports[0],
1809                         'egress': egress_ports[0],
1810                         'service_function_parameters': {
1811                             'correlation': correlation}}
1812             new_sfi = self.neutron.create_sfc_port_pair({'port_pair': sfi_dict})
1813             return new_sfi['port_pair']['id']
1814         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1815                 neExceptions.NeutronException, ConnectionError) as e:
1816             if new_sfi:
1817                 try:
1818                     self.neutron.delete_sfc_port_pair(
1819                         new_sfi['port_pair']['id'])
1820                 except Exception:
1821                     self.logger.error(
1822                         'Creation of Service Function Instance failed, with '
1823                         'subsequent deletion failure as well.')
1824             self._format_exception(e)
1825
1826     def get_sfi(self, sfi_id):
1827         self.logger.debug(
1828             'Getting Service Function Instance %s from VIM', sfi_id)
1829         filter_dict = {"id": sfi_id}
1830         sfi_list = self.get_sfi_list(filter_dict)
1831         if len(sfi_list) == 0:
1832             raise vimconn.vimconnNotFoundException(
1833                 "Service Function Instance '{}' not found".format(sfi_id))
1834         elif len(sfi_list) > 1:
1835             raise vimconn.vimconnConflictException(
1836                 'Found more than one Service Function Instance '
1837                 'with this criteria')
1838         sfi = sfi_list[0]
1839         return sfi
1840
1841     def get_sfi_list(self, filter_dict={}):
1842         self.logger.debug("Getting Service Function Instances from "
1843                           "VIM filter: '%s'", str(filter_dict))
1844         try:
1845             self._reload_connection()
1846             filter_dict_os = filter_dict.copy()
1847             if self.api_version3 and "tenant_id" in filter_dict_os:
1848                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1849             sfi_dict = self.neutron.list_sfc_port_pairs(**filter_dict_os)
1850             sfi_list = sfi_dict["port_pairs"]
1851             self.__sfi_os2mano(sfi_list)
1852             return sfi_list
1853         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1854                 neExceptions.NeutronException, ConnectionError) as e:
1855             self._format_exception(e)
1856
1857     def delete_sfi(self, sfi_id):
1858         self.logger.debug("Deleting Service Function Instance '%s' "
1859                           "from VIM", sfi_id)
1860         try:
1861             self._reload_connection()
1862             self.neutron.delete_sfc_port_pair(sfi_id)
1863             return sfi_id
1864         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1865                 ksExceptions.ClientException, neExceptions.NeutronException,
1866                 ConnectionError) as e:
1867             self._format_exception(e)
1868
1869     def new_sf(self, name, sfis, sfc_encap=True):
1870         self.logger.debug("Adding a new Service Function to VIM, "
1871                           "named '%s'", name)
1872         try:
1873             new_sf = None
1874             self._reload_connection()
1875             # correlation = None
1876             # if sfc_encap:
1877             #     correlation = 'nsh'
1878             for instance in sfis:
1879                 sfi = self.get_sfi(instance)
1880                 if sfi.get('sfc_encap') != sfc_encap:
1881                     raise vimconn.vimconnNotSupportedException(
1882                         "OpenStack VIM connector requires all SFIs of the "
1883                         "same SF to share the same SFC Encapsulation")
1884             sf_dict = {'name': name,
1885                        'port_pairs': sfis}
1886             new_sf = self.neutron.create_sfc_port_pair_group({
1887                 'port_pair_group': sf_dict})
1888             return new_sf['port_pair_group']['id']
1889         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1890                 neExceptions.NeutronException, ConnectionError) as e:
1891             if new_sf:
1892                 try:
1893                     self.neutron.delete_sfc_port_pair_group(
1894                         new_sf['port_pair_group']['id'])
1895                 except Exception:
1896                     self.logger.error(
1897                         'Creation of Service Function failed, with '
1898                         'subsequent deletion failure as well.')
1899             self._format_exception(e)
1900
1901     def get_sf(self, sf_id):
1902         self.logger.debug("Getting Service Function %s from VIM", sf_id)
1903         filter_dict = {"id": sf_id}
1904         sf_list = self.get_sf_list(filter_dict)
1905         if len(sf_list) == 0:
1906             raise vimconn.vimconnNotFoundException(
1907                 "Service Function '{}' not found".format(sf_id))
1908         elif len(sf_list) > 1:
1909             raise vimconn.vimconnConflictException(
1910                 "Found more than one Service Function with this criteria")
1911         sf = sf_list[0]
1912         return sf
1913
1914     def get_sf_list(self, filter_dict={}):
1915         self.logger.debug("Getting Service Function from VIM filter: '%s'",
1916                           str(filter_dict))
1917         try:
1918             self._reload_connection()
1919             filter_dict_os = filter_dict.copy()
1920             if self.api_version3 and "tenant_id" in filter_dict_os:
1921                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1922             sf_dict = self.neutron.list_sfc_port_pair_groups(**filter_dict_os)
1923             sf_list = sf_dict["port_pair_groups"]
1924             self.__sf_os2mano(sf_list)
1925             return sf_list
1926         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1927                 neExceptions.NeutronException, ConnectionError) as e:
1928             self._format_exception(e)
1929
1930     def delete_sf(self, sf_id):
1931         self.logger.debug("Deleting Service Function '%s' from VIM", sf_id)
1932         try:
1933             self._reload_connection()
1934             self.neutron.delete_sfc_port_pair_group(sf_id)
1935             return sf_id
1936         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1937                 ksExceptions.ClientException, neExceptions.NeutronException,
1938                 ConnectionError) as e:
1939             self._format_exception(e)
1940
1941     def new_sfp(self, name, classifications, sfs, sfc_encap=True, spi=None):
1942         self.logger.debug("Adding a new Service Function Path to VIM, "
1943                           "named '%s'", name)
1944         try:
1945             new_sfp = None
1946             self._reload_connection()
1947             # In networking-sfc the MPLS encapsulation is legacy
1948             # should be used when no full SFC Encapsulation is intended
1949             sfc_encap = 'mpls'
1950             if sfc_encap:
1951                 correlation = 'nsh'
1952             sfp_dict = {'name': name,
1953                         'flow_classifiers': classifications,
1954                         'port_pair_groups': sfs,
1955                         'chain_parameters': {'correlation': correlation}}
1956             if spi:
1957                 sfp_dict['chain_id'] = spi
1958             new_sfp = self.neutron.create_sfc_port_chain({'port_chain': sfp_dict})
1959             return new_sfp["port_chain"]["id"]
1960         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1961                 neExceptions.NeutronException, ConnectionError) as e:
1962             if new_sfp:
1963                 try:
1964                     self.neutron.delete_sfc_port_chain(new_sfp['port_chain']['id'])
1965                 except Exception:
1966                     self.logger.error(
1967                         'Creation of Service Function Path failed, with '
1968                         'subsequent deletion failure as well.')
1969             self._format_exception(e)
1970
1971     def get_sfp(self, sfp_id):
1972         self.logger.debug(" Getting Service Function Path %s from VIM", sfp_id)
1973         filter_dict = {"id": sfp_id}
1974         sfp_list = self.get_sfp_list(filter_dict)
1975         if len(sfp_list) == 0:
1976             raise vimconn.vimconnNotFoundException(
1977                 "Service Function Path '{}' not found".format(sfp_id))
1978         elif len(sfp_list) > 1:
1979             raise vimconn.vimconnConflictException(
1980                 "Found more than one Service Function Path with this criteria")
1981         sfp = sfp_list[0]
1982         return sfp
1983
1984     def get_sfp_list(self, filter_dict={}):
1985         self.logger.debug("Getting Service Function Paths from VIM filter: "
1986                           "'%s'", str(filter_dict))
1987         try:
1988             self._reload_connection()
1989             filter_dict_os = filter_dict.copy()
1990             if self.api_version3 and "tenant_id" in filter_dict_os:
1991                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1992             sfp_dict = self.neutron.list_sfc_port_chains(**filter_dict_os)
1993             sfp_list = sfp_dict["port_chains"]
1994             self.__sfp_os2mano(sfp_list)
1995             return sfp_list
1996         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1997                 neExceptions.NeutronException, ConnectionError) as e:
1998             self._format_exception(e)
1999
2000     def delete_sfp(self, sfp_id):
2001         self.logger.debug(
2002             "Deleting Service Function Path '%s' from VIM", sfp_id)
2003         try:
2004             self._reload_connection()
2005             self.neutron.delete_sfc_port_chain(sfp_id)
2006             return sfp_id
2007         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
2008                 ksExceptions.ClientException, neExceptions.NeutronException,
2009                 ConnectionError) as e:
2010             self._format_exception(e)