osm_ro/vimconn_openstack.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 osconnector implements all the methods to interact with openstack using the python-neutronclient.
  26
  27 For the VNF forwarding graph, The OpenStack VIM connector calls the
  28 networking-sfc Neutron extension methods, whose resources are mapped
  29 to the VIM connector's SFC resources as follows:
  30 - Classification (OSM) -> Flow Classifier (Neutron)
  31 - Service Function Instance (OSM) -> Port Pair (Neutron)
  32 - Service Function (OSM) -> Port Pair Group (Neutron)
  33 - Service Function Path (OSM) -> Port Chain (Neutron)
  34 '''
  35 __author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes, xFlow Research, Igor D.C., Eduardo Sousa"
  36 __date__  = "$22-sep-2017 23:59:59$"
  37
  38 import vimconn
  39 # import json
  40 import logging
  41 import netaddr
  42 import time
  43 import yaml
  44 import random
  45 import re
  46 import copy
  47
  48 from novaclient import client as nClient, exceptions as nvExceptions
  49 from keystoneauth1.identity import v2, v3
  50 from keystoneauth1 import session
  51 import keystoneclient.exceptions as ksExceptions
  52 import keystoneclient.v3.client as ksClient_v3
  53 import keystoneclient.v2_0.client as ksClient_v2
  54 from glanceclient import client as glClient
  55 import glanceclient.exc as gl1Exceptions
  56 from  cinderclient import client as cClient
  57 from httplib import HTTPException
  58 from neutronclient.neutron import client as neClient
  59 from neutronclient.common import exceptions as neExceptions
  60 from requests.exceptions import ConnectionError
  61
  62
  63 """contain the openstack virtual machine status to openmano status"""
  64 vmStatus2manoFormat={'ACTIVE':'ACTIVE',
  65                      'PAUSED':'PAUSED',
  66                      'SUSPENDED': 'SUSPENDED',
  67                      'SHUTOFF':'INACTIVE',
  68                      'BUILD':'BUILD',
  69                      'ERROR':'ERROR','DELETED':'DELETED'
  70                      }
  71 netStatus2manoFormat={'ACTIVE':'ACTIVE','PAUSED':'PAUSED','INACTIVE':'INACTIVE','BUILD':'BUILD','ERROR':'ERROR','DELETED':'DELETED'
  72                      }
  73
  74 supportedClassificationTypes = ['legacy_flow_classifier']
  75
  76 #global var to have a timeout creating and deleting volumes
  77 volume_timeout = 600
  78 server_timeout = 600
  79
  80 class vimconnector(vimconn.vimconnector):
  81     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None,
  82                  log_level=None, config={}, persistent_info={}):
  83         '''using common constructor parameters. In this case
  84         'url' is the keystone authorization url,
  85         'url_admin' is not use
  86         '''
  87         api_version = config.get('APIversion')
  88         if api_version and api_version not in ('v3.3', 'v2.0', '2', '3'):
  89             raise vimconn.vimconnException("Invalid value '{}' for config:APIversion. "
  90                                            "Allowed values are 'v3.3', 'v2.0', '2' or '3'".format(api_version))
  91         vim_type = config.get('vim_type')
  92         if vim_type and vim_type not in ('vio', 'VIO'):
  93             raise vimconn.vimconnException("Invalid value '{}' for config:vim_type."
  94                             "Allowed values are 'vio' or 'VIO'".format(vim_type))
  95
  96         if config.get('dataplane_net_vlan_range') is not None:
  97             #validate vlan ranges provided by user
  98             self._validate_vlan_ranges(config.get('dataplane_net_vlan_range'))
  99
 100         vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level,
 101                                       config)
 102
 103         if self.config.get("insecure") and self.config.get("ca_cert"):
 104             raise vimconn.vimconnException("options insecure and ca_cert are mutually exclusive")
 105         self.verify = True
 106         if self.config.get("insecure"):
 107             self.verify = False
 108         if self.config.get("ca_cert"):
 109             self.verify = self.config.get("ca_cert")
 110
 111         if not url:
 112             raise TypeError('url param can not be NoneType')
 113         self.persistent_info = persistent_info
 114         self.availability_zone = persistent_info.get('availability_zone', None)
 115         self.session = persistent_info.get('session', {'reload_client': True})
 116         self.nova = self.session.get('nova')
 117         self.neutron = self.session.get('neutron')
 118         self.cinder = self.session.get('cinder')
 119         self.glance = self.session.get('glance')
 120         # self.glancev1 = self.session.get('glancev1')
 121         self.keystone = self.session.get('keystone')
 122         self.api_version3 = self.session.get('api_version3')
 123         self.vim_type = self.config.get("vim_type")
 124         if self.vim_type:
 125             self.vim_type = self.vim_type.upper()
 126         if self.config.get("use_internal_endpoint"):
 127             self.endpoint_type = "internalURL"
 128         else:
 129             self.endpoint_type = None
 130
 131         self.logger = logging.getLogger('openmano.vim.openstack')
 132
 133         ####### VIO Specific Changes #########
 134         if self.vim_type == "VIO":
 135             self.logger = logging.getLogger('openmano.vim.vio')
 136
 137         if log_level:
 138             self.logger.setLevel( getattr(logging, log_level))
 139
 140     def __getitem__(self, index):
 141         """Get individuals parameters.
 142         Throw KeyError"""
 143         if index == 'project_domain_id':
 144             return self.config.get("project_domain_id")
 145         elif index == 'user_domain_id':
 146             return self.config.get("user_domain_id")
 147         else:
 148             return vimconn.vimconnector.__getitem__(self, index)
 149
 150     def __setitem__(self, index, value):
 151         """Set individuals parameters and it is marked as dirty so to force connection reload.
 152         Throw KeyError"""
 153         if index == 'project_domain_id':
 154             self.config["project_domain_id"] = value
 155         elif index == 'user_domain_id':
 156                 self.config["user_domain_id"] = value
 157         else:
 158             vimconn.vimconnector.__setitem__(self, index, value)
 159         self.session['reload_client'] = True
 160
 161     def _reload_connection(self):
 162         '''Called before any operation, it check if credentials has changed
 163         Throw keystoneclient.apiclient.exceptions.AuthorizationFailure
 164         '''
 165         #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-)
 166         if self.session['reload_client']:
 167             if self.config.get('APIversion'):
 168                 self.api_version3 = self.config['APIversion'] == 'v3.3' or self.config['APIversion'] == '3'
 169             else:  # get from ending auth_url that end with v3 or with v2.0
 170                 self.api_version3 =  self.url.endswith("/v3") or self.url.endswith("/v3/")
 171             self.session['api_version3'] = self.api_version3
 172             if self.api_version3:
 173                 if self.config.get('project_domain_id') or self.config.get('project_domain_name'):
 174                     project_domain_id_default = None
 175                 else:
 176                     project_domain_id_default = 'default'
 177                 if self.config.get('user_domain_id') or self.config.get('user_domain_name'):
 178                     user_domain_id_default = None
 179                 else:
 180                     user_domain_id_default = 'default'
 181                 auth = v3.Password(auth_url=self.url,
 182                                    username=self.user,
 183                                    password=self.passwd,
 184                                    project_name=self.tenant_name,
 185                                    project_id=self.tenant_id,
 186                                    project_domain_id=self.config.get('project_domain_id', project_domain_id_default),
 187                                    user_domain_id=self.config.get('user_domain_id', user_domain_id_default),
 188                                    project_domain_name=self.config.get('project_domain_name'),
 189                                    user_domain_name=self.config.get('user_domain_name'))
 190             else:
 191                 auth = v2.Password(auth_url=self.url,
 192                                    username=self.user,
 193                                    password=self.passwd,
 194                                    tenant_name=self.tenant_name,
 195                                    tenant_id=self.tenant_id)
 196             sess = session.Session(auth=auth, verify=self.verify)
 197             if self.api_version3:
 198                 self.keystone = ksClient_v3.Client(session=sess, endpoint_type=self.endpoint_type)
 199             else:
 200                 self.keystone = ksClient_v2.Client(session=sess, endpoint_type=self.endpoint_type)
 201             self.session['keystone'] = self.keystone
 202             # In order to enable microversion functionality an explicit microversion must be specified in 'config'.
 203             # This implementation approach is due to the warning message in
 204             # https://developer.openstack.org/api-guide/compute/microversions.html
 205             # where it is stated that microversion backwards compatibility is not guaranteed and clients should
 206             # always require an specific microversion.
 207             # To be able to use 'device role tagging' functionality define 'microversion: 2.32' in datacenter config
 208             version = self.config.get("microversion")
 209             if not version:
 210                 version = "2.1"
 211             self.nova = self.session['nova'] = nClient.Client(str(version), session=sess, endpoint_type=self.endpoint_type)
 212             self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess, endpoint_type=self.endpoint_type)
 213             self.cinder = self.session['cinder'] = cClient.Client(2, session=sess, endpoint_type=self.endpoint_type)
 214             if self.endpoint_type == "internalURL":
 215                 glance_service_id = self.keystone.services.list(name="glance")[0].id
 216                 glance_endpoint = self.keystone.endpoints.list(glance_service_id, interface="internal")[0].url
 217             else:
 218                 glance_endpoint = None
 219             self.glance = self.session['glance'] = glClient.Client(2, session=sess, endpoint=glance_endpoint)
 220             #using version 1 of glance client in new_image()
 221             # self.glancev1 = self.session['glancev1'] = glClient.Client('1', session=sess,
 222             #                                                            endpoint=glance_endpoint)
 223             self.session['reload_client'] = False
 224             self.persistent_info['session'] = self.session
 225             # add availablity zone info inside  self.persistent_info
 226             self._set_availablity_zones()
 227             self.persistent_info['availability_zone'] = self.availability_zone
 228
 229     def __net_os2mano(self, net_list_dict):
 230         '''Transform the net openstack format to mano format
 231         net_list_dict can be a list of dict or a single dict'''
 232         if type(net_list_dict) is dict:
 233             net_list_=(net_list_dict,)
 234         elif type(net_list_dict) is list:
 235             net_list_=net_list_dict
 236         else:
 237             raise TypeError("param net_list_dict must be a list or a dictionary")
 238         for net in net_list_:
 239             if net.get('provider:network_type') == "vlan":
 240                 net['type']='data'
 241             else:
 242                 net['type']='bridge'
 243
 244     def __classification_os2mano(self, class_list_dict):
 245         """Transform the openstack format (Flow Classifier) to mano format
 246         (Classification) class_list_dict can be a list of dict or a single dict
 247         """
 248         if isinstance(class_list_dict, dict):
 249             class_list_ = [class_list_dict]
 250         elif isinstance(class_list_dict, list):
 251             class_list_ = class_list_dict
 252         else:
 253             raise TypeError(
 254                 "param class_list_dict must be a list or a dictionary")
 255         for classification in class_list_:
 256             id = classification.pop('id')
 257             name = classification.pop('name')
 258             description = classification.pop('description')
 259             project_id = classification.pop('project_id')
 260             tenant_id = classification.pop('tenant_id')
 261             original_classification = copy.deepcopy(classification)
 262             classification.clear()
 263             classification['ctype'] = 'legacy_flow_classifier'
 264             classification['definition'] = original_classification
 265             classification['id'] = id
 266             classification['name'] = name
 267             classification['description'] = description
 268             classification['project_id'] = project_id
 269             classification['tenant_id'] = tenant_id
 270
 271     def __sfi_os2mano(self, sfi_list_dict):
 272         """Transform the openstack format (Port Pair) to mano format (SFI)
 273         sfi_list_dict can be a list of dict or a single dict
 274         """
 275         if isinstance(sfi_list_dict, dict):
 276             sfi_list_ = [sfi_list_dict]
 277         elif isinstance(sfi_list_dict, list):
 278             sfi_list_ = sfi_list_dict
 279         else:
 280             raise TypeError(
 281                 "param sfi_list_dict must be a list or a dictionary")
 282         for sfi in sfi_list_:
 283             sfi['ingress_ports'] = []
 284             sfi['egress_ports'] = []
 285             if sfi.get('ingress'):
 286                 sfi['ingress_ports'].append(sfi['ingress'])
 287             if sfi.get('egress'):
 288                 sfi['egress_ports'].append(sfi['egress'])
 289             del sfi['ingress']
 290             del sfi['egress']
 291             params = sfi.get('service_function_parameters')
 292             sfc_encap = False
 293             if params:
 294                 correlation = params.get('correlation')
 295                 if correlation:
 296                     sfc_encap = True
 297             sfi['sfc_encap'] = sfc_encap
 298             del sfi['service_function_parameters']
 299
 300     def __sf_os2mano(self, sf_list_dict):
 301         """Transform the openstack format (Port Pair Group) to mano format (SF)
 302         sf_list_dict can be a list of dict or a single dict
 303         """
 304         if isinstance(sf_list_dict, dict):
 305             sf_list_ = [sf_list_dict]
 306         elif isinstance(sf_list_dict, list):
 307             sf_list_ = sf_list_dict
 308         else:
 309             raise TypeError(
 310                 "param sf_list_dict must be a list or a dictionary")
 311         for sf in sf_list_:
 312             del sf['port_pair_group_parameters']
 313             sf['sfis'] = sf['port_pairs']
 314             del sf['port_pairs']
 315
 316     def __sfp_os2mano(self, sfp_list_dict):
 317         """Transform the openstack format (Port Chain) to mano format (SFP)
 318         sfp_list_dict can be a list of dict or a single dict
 319         """
 320         if isinstance(sfp_list_dict, dict):
 321             sfp_list_ = [sfp_list_dict]
 322         elif isinstance(sfp_list_dict, list):
 323             sfp_list_ = sfp_list_dict
 324         else:
 325             raise TypeError(
 326                 "param sfp_list_dict must be a list or a dictionary")
 327         for sfp in sfp_list_:
 328             params = sfp.pop('chain_parameters')
 329             sfc_encap = False
 330             if params:
 331                 correlation = params.get('correlation')
 332                 if correlation:
 333                     sfc_encap = True
 334             sfp['sfc_encap'] = sfc_encap
 335             sfp['spi'] = sfp.pop('chain_id')
 336             sfp['classifications'] = sfp.pop('flow_classifiers')
 337             sfp['service_functions'] = sfp.pop('port_pair_groups')
 338
 339     # placeholder for now; read TODO note below
 340     def _validate_classification(self, type, definition):
 341         # only legacy_flow_classifier Type is supported at this point
 342         return True
 343         # TODO(igordcard): this method should be an abstract method of an
 344         # abstract Classification class to be implemented by the specific
 345         # Types. Also, abstract vimconnector should call the validation
 346         # method before the implemented VIM connectors are called.
 347
 348     def _format_exception(self, exception):
 349         '''Transform a keystone, nova, neutron  exception into a vimconn exception'''
 350         if isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError,
 351                                   ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed
 352                                   )):
 353             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 354         elif isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound)):
 355             raise vimconn.vimconnNotFoundException(type(exception).__name__ + ": " + str(exception))
 356         elif isinstance(exception,  (KeyError, nvExceptions.BadRequest)):
 357             raise vimconn.vimconnException(type(exception).__name__ + ": " + str(exception))
 358         elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException,
 359                                     neExceptions.NeutronException)):
 360             raise vimconn.vimconnUnexpectedResponse(type(exception).__name__ + ": " + str(exception))
 361         elif isinstance(exception, nvExceptions.Conflict):
 362             raise vimconn.vimconnConflictException(type(exception).__name__ + ": " + str(exception))
 363         elif isinstance(exception, vimconn.vimconnException):
 364             raise exception
 365         else:  # ()
 366             self.logger.error("General Exception " + str(exception), exc_info=True)
 367             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 368
 369     def get_tenant_list(self, filter_dict={}):
 370         '''Obtain tenants of VIM
 371         filter_dict can contain the following keys:
 372             name: filter by tenant name
 373             id: filter by tenant uuid/id
 374             <other VIM specific>
 375         Returns the tenant list of dictionaries: [{'name':'<name>, 'id':'<id>, ...}, ...]
 376         '''
 377         self.logger.debug("Getting tenants from VIM filter: '%s'", str(filter_dict))
 378         try:
 379             self._reload_connection()
 380             if self.api_version3:
 381                 project_class_list = self.keystone.projects.list(name=filter_dict.get("name"))
 382             else:
 383                 project_class_list = self.keystone.tenants.findall(**filter_dict)
 384             project_list=[]
 385             for project in project_class_list:
 386                 if filter_dict.get('id') and filter_dict["id"] != project.id:
 387                     continue
 388                 project_list.append(project.to_dict())
 389             return project_list
 390         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError) as e:
 391             self._format_exception(e)
 392
 393     def new_tenant(self, tenant_name, tenant_description):
 394         '''Adds a new tenant to openstack VIM. Returns the tenant identifier'''
 395         self.logger.debug("Adding a new tenant name: %s", tenant_name)
 396         try:
 397             self._reload_connection()
 398             if self.api_version3:
 399                 project = self.keystone.projects.create(tenant_name, self.config.get("project_domain_id", "default"),
 400                                                         description=tenant_description, is_domain=False)
 401             else:
 402                 project = self.keystone.tenants.create(tenant_name, tenant_description)
 403             return project.id
 404         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 405             self._format_exception(e)
 406
 407     def delete_tenant(self, tenant_id):
 408         '''Delete a tenant from openstack VIM. Returns the old tenant identifier'''
 409         self.logger.debug("Deleting tenant %s from VIM", tenant_id)
 410         try:
 411             self._reload_connection()
 412             if self.api_version3:
 413                 self.keystone.projects.delete(tenant_id)
 414             else:
 415                 self.keystone.tenants.delete(tenant_id)
 416             return tenant_id
 417         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 418             self._format_exception(e)
 419
 420     def new_network(self,net_name, net_type, ip_profile=None, shared=False, vlan=None):
 421         '''Adds a tenant network to VIM. Returns the network identifier'''
 422         self.logger.debug("Adding a new network to VIM name '%s', type '%s'", net_name, net_type)
 423         #self.logger.debug(">>>>>>>>>>>>>>>>>> IP profile %s", str(ip_profile))
 424         try:
 425             new_net = None
 426             self._reload_connection()
 427             network_dict = {'name': net_name, 'admin_state_up': True}
 428             if net_type=="data" or net_type=="ptp":
 429                 if self.config.get('dataplane_physical_net') == None:
 430                     raise vimconn.vimconnConflictException("You must provide a 'dataplane_physical_net' at config value before creating sriov network")
 431                 network_dict["provider:physical_network"] = self.config['dataplane_physical_net'] #"physnet_sriov" #TODO physical
 432                 network_dict["provider:network_type"]     = "vlan"
 433                 if vlan!=None:
 434                     network_dict["provider:network_type"] = vlan
 435
 436                 ####### VIO Specific Changes #########
 437                 if self.vim_type == "VIO":
 438                     if vlan is not None:
 439                         network_dict["provider:segmentation_id"] = vlan
 440                     else:
 441                         if self.config.get('dataplane_net_vlan_range') is None:
 442                             raise vimconn.vimconnConflictException("You must provide "\
 443                                 "'dataplane_net_vlan_range' in format [start_ID - end_ID]"\
 444                                 "at config value before creating sriov network with vlan tag")
 445
 446                         network_dict["provider:segmentation_id"] = self._genrate_vlanID()
 447
 448             network_dict["shared"]=shared
 449             new_net=self.neutron.create_network({'network':network_dict})
 450             #print new_net
 451             #create subnetwork, even if there is no profile
 452             if not ip_profile:
 453                 ip_profile = {}
 454             if not ip_profile.get('subnet_address'):
 455                 #Fake subnet is required
 456                 subnet_rand = random.randint(0, 255)
 457                 ip_profile['subnet_address'] = "192.168.{}.0/24".format(subnet_rand)
 458             if 'ip_version' not in ip_profile:
 459                 ip_profile['ip_version'] = "IPv4"
 460             subnet = {"name":net_name+"-subnet",
 461                     "network_id": new_net["network"]["id"],
 462                     "ip_version": 4 if ip_profile['ip_version']=="IPv4" else 6,
 463                     "cidr": ip_profile['subnet_address']
 464                     }
 465             # Gateway should be set to None if not needed. Otherwise openstack assigns one by default
 466             if ip_profile.get('gateway_address'):
 467                 subnet['gateway_ip'] = ip_profile['gateway_address']
 468             else:
 469                 subnet['gateway_ip'] = None
 470             if ip_profile.get('dns_address'):
 471                 subnet['dns_nameservers'] = ip_profile['dns_address'].split(";")
 472             if 'dhcp_enabled' in ip_profile:
 473                 subnet['enable_dhcp'] = False if \
 474                     ip_profile['dhcp_enabled']=="false" or ip_profile['dhcp_enabled']==False else True
 475             if ip_profile.get('dhcp_start_address'):
 476                 subnet['allocation_pools'] = []
 477                 subnet['allocation_pools'].append(dict())
 478                 subnet['allocation_pools'][0]['start'] = ip_profile['dhcp_start_address']
 479             if ip_profile.get('dhcp_count'):
 480                 #parts = ip_profile['dhcp_start_address'].split('.')
 481                 #ip_int = (int(parts[0]) << 24) + (int(parts[1]) << 16) + (int(parts[2]) << 8) + int(parts[3])
 482                 ip_int = int(netaddr.IPAddress(ip_profile['dhcp_start_address']))
 483                 ip_int += ip_profile['dhcp_count'] - 1
 484                 ip_str = str(netaddr.IPAddress(ip_int))
 485                 subnet['allocation_pools'][0]['end'] = ip_str
 486             #self.logger.debug(">>>>>>>>>>>>>>>>>> Subnet: %s", str(subnet))
 487             self.neutron.create_subnet({"subnet": subnet} )
 488             return new_net["network"]["id"]
 489         except Exception as e:
 490             if new_net:
 491                 self.neutron.delete_network(new_net['network']['id'])
 492             self._format_exception(e)
 493
 494     def get_network_list(self, filter_dict={}):
 495         '''Obtain tenant networks of VIM
 496         Filter_dict can be:
 497             name: network name
 498             id: network uuid
 499             shared: boolean
 500             tenant_id: tenant
 501             admin_state_up: boolean
 502             status: 'ACTIVE'
 503         Returns the network list of dictionaries
 504         '''
 505         self.logger.debug("Getting network from VIM filter: '%s'", str(filter_dict))
 506         try:
 507             self._reload_connection()
 508             filter_dict_os = filter_dict.copy()
 509             if self.api_version3 and "tenant_id" in filter_dict_os:
 510                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')  #T ODO check
 511             net_dict = self.neutron.list_networks(**filter_dict_os)
 512             net_list = net_dict["networks"]
 513             self.__net_os2mano(net_list)
 514             return net_list
 515         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 516             self._format_exception(e)
 517
 518     def get_network(self, net_id):
 519         '''Obtain details of network from VIM
 520         Returns the network information from a network id'''
 521         self.logger.debug(" Getting tenant network %s from VIM", net_id)
 522         filter_dict={"id": net_id}
 523         net_list = self.get_network_list(filter_dict)
 524         if len(net_list)==0:
 525             raise vimconn.vimconnNotFoundException("Network '{}' not found".format(net_id))
 526         elif len(net_list)>1:
 527             raise vimconn.vimconnConflictException("Found more than one network with this criteria")
 528         net = net_list[0]
 529         subnets=[]
 530         for subnet_id in net.get("subnets", () ):
 531             try:
 532                 subnet = self.neutron.show_subnet(subnet_id)
 533             except Exception as e:
 534                 self.logger.error("osconnector.get_network(): Error getting subnet %s %s" % (net_id, str(e)))
 535                 subnet = {"id": subnet_id, "fault": str(e)}
 536             subnets.append(subnet)
 537         net["subnets"] = subnets
 538         net["encapsulation"] = net.get('provider:network_type')
 539         net["segmentation_id"] = net.get('provider:segmentation_id')
 540         return net
 541
 542     def delete_network(self, net_id):
 543         '''Deletes a tenant network from VIM. Returns the old network identifier'''
 544         self.logger.debug("Deleting network '%s' from VIM", net_id)
 545         try:
 546             self._reload_connection()
 547             #delete VM ports attached to this networks before the network
 548             ports = self.neutron.list_ports(network_id=net_id)
 549             for p in ports['ports']:
 550                 try:
 551                     self.neutron.delete_port(p["id"])
 552                 except Exception as e:
 553                     self.logger.error("Error deleting port %s: %s", p["id"], str(e))
 554             self.neutron.delete_network(net_id)
 555             return net_id
 556         except (neExceptions.ConnectionFailed, neExceptions.NetworkNotFoundClient, neExceptions.NeutronException,
 557                 ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 558             self._format_exception(e)
 559
 560     def refresh_nets_status(self, net_list):
 561         '''Get the status of the networks
 562            Params: the list of network identifiers
 563            Returns a dictionary with:
 564                 net_id:         #VIM id of this network
 565                     status:     #Mandatory. Text with one of:
 566                                 #  DELETED (not found at vim)
 567                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 568                                 #  OTHER (Vim reported other status not understood)
 569                                 #  ERROR (VIM indicates an ERROR status)
 570                                 #  ACTIVE, INACTIVE, DOWN (admin down),
 571                                 #  BUILD (on building process)
 572                                 #
 573                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 574                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 575
 576         '''
 577         net_dict={}
 578         for net_id in net_list:
 579             net = {}
 580             try:
 581                 net_vim = self.get_network(net_id)
 582                 if net_vim['status'] in netStatus2manoFormat:
 583                     net["status"] = netStatus2manoFormat[ net_vim['status'] ]
 584                 else:
 585                     net["status"] = "OTHER"
 586                     net["error_msg"] = "VIM status reported " + net_vim['status']
 587
 588                 if net['status'] == "ACTIVE" and not net_vim['admin_state_up']:
 589                     net['status'] = 'DOWN'
 590                 try:
 591                     net['vim_info'] = yaml.safe_dump(net_vim, default_flow_style=True, width=256)
 592                 except yaml.representer.RepresenterError:
 593                     net['vim_info'] = str(net_vim)
 594                 if net_vim.get('fault'):  #TODO
 595                     net['error_msg'] = str(net_vim['fault'])
 596             except vimconn.vimconnNotFoundException as e:
 597                 self.logger.error("Exception getting net status: %s", str(e))
 598                 net['status'] = "DELETED"
 599                 net['error_msg'] = str(e)
 600             except vimconn.vimconnException as e:
 601                 self.logger.error("Exception getting net status: %s", str(e))
 602                 net['status'] = "VIM_ERROR"
 603                 net['error_msg'] = str(e)
 604             net_dict[net_id] = net
 605         return net_dict
 606
 607     def get_flavor(self, flavor_id):
 608         '''Obtain flavor details from the  VIM. Returns the flavor dict details'''
 609         self.logger.debug("Getting flavor '%s'", flavor_id)
 610         try:
 611             self._reload_connection()
 612             flavor = self.nova.flavors.find(id=flavor_id)
 613             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 614             return flavor.to_dict()
 615         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 616             self._format_exception(e)
 617
 618     def get_flavor_id_from_data(self, flavor_dict):
 619         """Obtain flavor id that match the flavor description
 620            Returns the flavor_id or raises a vimconnNotFoundException
 621            flavor_dict: contains the required ram, vcpus, disk
 622            If 'use_existing_flavors' is set to True at config, the closer flavor that provides same or more ram, vcpus
 623                 and disk is returned. Otherwise a flavor with exactly same ram, vcpus and disk is returned or a
 624                 vimconnNotFoundException is raised
 625         """
 626         exact_match = False if self.config.get('use_existing_flavors') else True
 627         try:
 628             self._reload_connection()
 629             flavor_candidate_id = None
 630             flavor_candidate_data = (10000, 10000, 10000)
 631             flavor_target = (flavor_dict["ram"], flavor_dict["vcpus"], flavor_dict["disk"])
 632             # numa=None
 633             numas = flavor_dict.get("extended", {}).get("numas")
 634             if numas:
 635                 #TODO
 636                 raise vimconn.vimconnNotFoundException("Flavor with EPA still not implemted")
 637                 # if len(numas) > 1:
 638                 #     raise vimconn.vimconnNotFoundException("Cannot find any flavor with more than one numa")
 639                 # numa=numas[0]
 640                 # numas = extended.get("numas")
 641             for flavor in self.nova.flavors.list():
 642                 epa = flavor.get_keys()
 643                 if epa:
 644                     continue
 645                     # TODO
 646                 flavor_data = (flavor.ram, flavor.vcpus, flavor.disk)
 647                 if flavor_data == flavor_target:
 648                     return flavor.id
 649                 elif not exact_match and flavor_target < flavor_data < flavor_candidate_data:
 650                     flavor_candidate_id = flavor.id
 651                     flavor_candidate_data = flavor_data
 652             if not exact_match and flavor_candidate_id:
 653                 return flavor_candidate_id
 654             raise vimconn.vimconnNotFoundException("Cannot find any flavor matching '{}'".format(str(flavor_dict)))
 655         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 656             self._format_exception(e)
 657
 658     def new_flavor(self, flavor_data, change_name_if_used=True):
 659         '''Adds a tenant flavor to openstack VIM
 660         if change_name_if_used is True, it will change name in case of conflict, because it is not supported name repetition
 661         Returns the flavor identifier
 662         '''
 663         self.logger.debug("Adding flavor '%s'", str(flavor_data))
 664         retry=0
 665         max_retries=3
 666         name_suffix = 0
 667         try:
 668             name=flavor_data['name']
 669             while retry<max_retries:
 670                 retry+=1
 671                 try:
 672                     self._reload_connection()
 673                     if change_name_if_used:
 674                         #get used names
 675                         fl_names=[]
 676                         fl=self.nova.flavors.list()
 677                         for f in fl:
 678                             fl_names.append(f.name)
 679                         while name in fl_names:
 680                             name_suffix += 1
 681                             name = flavor_data['name']+"-" + str(name_suffix)
 682
 683                     ram = flavor_data.get('ram',64)
 684                     vcpus = flavor_data.get('vcpus',1)
 685                     numa_properties=None
 686
 687                     extended = flavor_data.get("extended")
 688                     if extended:
 689                         numas=extended.get("numas")
 690                         if numas:
 691                             numa_nodes = len(numas)
 692                             if numa_nodes > 1:
 693                                 return -1, "Can not add flavor with more than one numa"
 694                             numa_properties = {"hw:numa_nodes":str(numa_nodes)}
 695                             numa_properties["hw:mem_page_size"] = "large"
 696                             numa_properties["hw:cpu_policy"] = "dedicated"
 697                             numa_properties["hw:numa_mempolicy"] = "strict"
 698                             if self.vim_type == "VIO":
 699                                 numa_properties["vmware:extra_config"] = '{"numa.nodeAffinity":"0"}'
 700                                 numa_properties["vmware:latency_sensitivity_level"] = "high"
 701                             for numa in numas:
 702                                 #overwrite ram and vcpus
 703                                 #check if key 'memory' is present in numa else use ram value at flavor
 704                                 if 'memory' in numa:
 705                                     ram = numa['memory']*1024
 706                                 #See for reference: https://specs.openstack.org/openstack/nova-specs/specs/mitaka/implemented/virt-driver-cpu-thread-pinning.html
 707                                 if 'paired-threads' in numa:
 708                                     vcpus = numa['paired-threads']*2
 709                                     #cpu_thread_policy "require" implies that the compute node must have an STM architecture
 710                                     numa_properties["hw:cpu_thread_policy"] = "require"
 711                                     numa_properties["hw:cpu_policy"] = "dedicated"
 712                                 elif 'cores' in numa:
 713                                     vcpus = numa['cores']
 714                                     # cpu_thread_policy "prefer" implies that the host must not have an SMT architecture, or a non-SMT architecture will be emulated
 715                                     numa_properties["hw:cpu_thread_policy"] = "isolate"
 716                                     numa_properties["hw:cpu_policy"] = "dedicated"
 717                                 elif 'threads' in numa:
 718                                     vcpus = numa['threads']
 719                                     # cpu_thread_policy "prefer" implies that the host may or may not have an SMT architecture
 720                                     numa_properties["hw:cpu_thread_policy"] = "prefer"
 721                                     numa_properties["hw:cpu_policy"] = "dedicated"
 722                                 # for interface in numa.get("interfaces",() ):
 723                                 #     if interface["dedicated"]=="yes":
 724                                 #         raise vimconn.vimconnException("Passthrough interfaces are not supported for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable)
 725                                 #     #TODO, add the key 'pci_passthrough:alias"="<label at config>:<number ifaces>"' when a way to connect it is available
 726
 727                     #create flavor
 728                     new_flavor=self.nova.flavors.create(name,
 729                                     ram,
 730                                     vcpus,
 731                                     flavor_data.get('disk',0),
 732                                     is_public=flavor_data.get('is_public', True)
 733                                 )
 734                     #add metadata
 735                     if numa_properties:
 736                         new_flavor.set_keys(numa_properties)
 737                     return new_flavor.id
 738                 except nvExceptions.Conflict as e:
 739                     if change_name_if_used and retry < max_retries:
 740                         continue
 741                     self._format_exception(e)
 742         #except nvExceptions.BadRequest as e:
 743         except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError, KeyError) as e:
 744             self._format_exception(e)
 745
 746     def delete_flavor(self,flavor_id):
 747         '''Deletes a tenant flavor from openstack VIM. Returns the old flavor_id
 748         '''
 749         try:
 750             self._reload_connection()
 751             self.nova.flavors.delete(flavor_id)
 752             return flavor_id
 753         #except nvExceptions.BadRequest as e:
 754         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 755             self._format_exception(e)
 756
 757     def new_image(self,image_dict):
 758         '''
 759         Adds a tenant image to VIM. imge_dict is a dictionary with:
 760             name: name
 761             disk_format: qcow2, vhd, vmdk, raw (by default), ...
 762             location: path or URI
 763             public: "yes" or "no"
 764             metadata: metadata of the image
 765         Returns the image_id
 766         '''
 767         retry=0
 768         max_retries=3
 769         while retry<max_retries:
 770             retry+=1
 771             try:
 772                 self._reload_connection()
 773                 #determine format  http://docs.openstack.org/developer/glance/formats.html
 774                 if "disk_format" in image_dict:
 775                     disk_format=image_dict["disk_format"]
 776                 else: #autodiscover based on extension
 777                     if image_dict['location'].endswith(".qcow2"):
 778                         disk_format="qcow2"
 779                     elif image_dict['location'].endswith(".vhd"):
 780                         disk_format="vhd"
 781                     elif image_dict['location'].endswith(".vmdk"):
 782                         disk_format="vmdk"
 783                     elif image_dict['location'].endswith(".vdi"):
 784                         disk_format="vdi"
 785                     elif image_dict['location'].endswith(".iso"):
 786                         disk_format="iso"
 787                     elif image_dict['location'].endswith(".aki"):
 788                         disk_format="aki"
 789                     elif image_dict['location'].endswith(".ari"):
 790                         disk_format="ari"
 791                     elif image_dict['location'].endswith(".ami"):
 792                         disk_format="ami"
 793                     else:
 794                         disk_format="raw"
 795                 self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location'])
 796                 new_image = self.glance.images.create(name=image_dict['name'])
 797                 if image_dict['location'].startswith("http"):
 798                     # TODO there is not a method to direct download. It must be downloaded locally with requests
 799                     raise vimconn.vimconnNotImplemented("Cannot create image from URL")
 800                 else: #local path
 801                     with open(image_dict['location']) as fimage:
 802                         self.glance.images.upload(new_image.id, fimage)
 803                         #new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 804                         #    container_format="bare", data=fimage, disk_format=disk_format)
 805                 metadata_to_load = image_dict.get('metadata')
 806                 #TODO location is a reserved word for current openstack versions. Use another word
 807                 metadata_to_load['location'] = image_dict['location']
 808                 self.glance.images.update(new_image.id, **metadata_to_load)
 809                 return new_image.id
 810             except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e:
 811                 self._format_exception(e)
 812             except (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 813                 if retry==max_retries:
 814                     continue
 815                 self._format_exception(e)
 816             except IOError as e:  #can not open the file
 817                 raise vimconn.vimconnConnectionException(type(e).__name__ + ": " + str(e)+ " for " + image_dict['location'],
 818                                                          http_code=vimconn.HTTP_Bad_Request)
 819
 820     def delete_image(self, image_id):
 821         '''Deletes a tenant image from openstack VIM. Returns the old id
 822         '''
 823         try:
 824             self._reload_connection()
 825             self.glance.images.delete(image_id)
 826             return image_id
 827         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e: #TODO remove
 828             self._format_exception(e)
 829
 830     def get_image_id_from_path(self, path):
 831         '''Get the image id from image path in the VIM database. Returns the image_id'''
 832         try:
 833             self._reload_connection()
 834             images = self.glance.images.list()
 835             for image in images:
 836                 if image.metadata.get("location")==path:
 837                     return image.id
 838             raise vimconn.vimconnNotFoundException("image with location '{}' not found".format( path))
 839         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 840             self._format_exception(e)
 841
 842     def get_image_list(self, filter_dict={}):
 843         '''Obtain tenant images from VIM
 844         Filter_dict can be:
 845             id: image id
 846             name: image name
 847             checksum: image checksum
 848         Returns the image list of dictionaries:
 849             [{<the fields at Filter_dict plus some VIM specific>}, ...]
 850             List can be empty
 851         '''
 852         self.logger.debug("Getting image list from VIM filter: '%s'", str(filter_dict))
 853         try:
 854             self._reload_connection()
 855             filter_dict_os = filter_dict.copy()
 856             #First we filter by the available filter fields: name, id. The others are removed.
 857             image_list = self.glance.images.list()
 858             filtered_list = []
 859             for image in image_list:
 860                 try:
 861                     if filter_dict.get("name") and image["name"] != filter_dict["name"]:
 862                         continue
 863                     if filter_dict.get("id") and image["id"] != filter_dict["id"]:
 864                         continue
 865                     if filter_dict.get("checksum") and image["checksum"] != filter_dict["checksum"]:
 866                         continue
 867
 868                     filtered_list.append(image.copy())
 869                 except gl1Exceptions.HTTPNotFound:
 870                     pass
 871             return filtered_list
 872         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 873             self._format_exception(e)
 874
 875     def __wait_for_vm(self, vm_id, status):
 876         """wait until vm is in the desired status and return True.
 877         If the VM gets in ERROR status, return false.
 878         If the timeout is reached generate an exception"""
 879         elapsed_time = 0
 880         while elapsed_time < server_timeout:
 881             vm_status = self.nova.servers.get(vm_id).status
 882             if vm_status == status:
 883                 return True
 884             if vm_status == 'ERROR':
 885                 return False
 886             time.sleep(5)
 887             elapsed_time += 5
 888
 889         # if we exceeded the timeout rollback
 890         if elapsed_time >= server_timeout:
 891             raise vimconn.vimconnException('Timeout waiting for instance ' + vm_id + ' to get ' + status,
 892                                            http_code=vimconn.HTTP_Request_Timeout)
 893
 894     def _get_openstack_availablity_zones(self):
 895         """
 896         Get from openstack availability zones available
 897         :return:
 898         """
 899         try:
 900             openstack_availability_zone = self.nova.availability_zones.list()
 901             openstack_availability_zone = [str(zone.zoneName) for zone in openstack_availability_zone
 902                                            if zone.zoneName != 'internal']
 903             return openstack_availability_zone
 904         except Exception as e:
 905             return None
 906
 907     def _set_availablity_zones(self):
 908         """
 909         Set vim availablity zone
 910         :return:
 911         """
 912
 913         if 'availability_zone' in self.config:
 914             vim_availability_zones = self.config.get('availability_zone')
 915             if isinstance(vim_availability_zones, str):
 916                 self.availability_zone = [vim_availability_zones]
 917             elif isinstance(vim_availability_zones, list):
 918                 self.availability_zone = vim_availability_zones
 919         else:
 920             self.availability_zone = self._get_openstack_availablity_zones()
 921
 922     def _get_vm_availability_zone(self, availability_zone_index, availability_zone_list):
 923         """
 924         Return thge availability zone to be used by the created VM.
 925         :return: The VIM availability zone to be used or None
 926         """
 927         if availability_zone_index is None:
 928             if not self.config.get('availability_zone'):
 929                 return None
 930             elif isinstance(self.config.get('availability_zone'), str):
 931                 return self.config['availability_zone']
 932             else:
 933                 # TODO consider using a different parameter at config for default AV and AV list match
 934                 return self.config['availability_zone'][0]
 935
 936         vim_availability_zones = self.availability_zone
 937         # check if VIM offer enough availability zones describe in the VNFD
 938         if vim_availability_zones and len(availability_zone_list) <= len(vim_availability_zones):
 939             # check if all the names of NFV AV match VIM AV names
 940             match_by_index = False
 941             for av in availability_zone_list:
 942                 if av not in vim_availability_zones:
 943                     match_by_index = True
 944                     break
 945             if match_by_index:
 946                 return vim_availability_zones[availability_zone_index]
 947             else:
 948                 return availability_zone_list[availability_zone_index]
 949         else:
 950             raise vimconn.vimconnConflictException("No enough availability zones at VIM for this deployment")
 951
 952     def new_vminstance(self, name, description, start, image_id, flavor_id, net_list, cloud_config=None, disk_list=None,
 953                        availability_zone_index=None, availability_zone_list=None):
 954         """Adds a VM instance to VIM
 955         Params:
 956             start: indicates if VM must start or boot in pause mode. Ignored
 957             image_id,flavor_id: iamge and flavor uuid
 958             net_list: list of interfaces, each one is a dictionary with:
 959                 name:
 960                 net_id: network uuid to connect
 961                 vpci: virtual vcpi to assign, ignored because openstack lack #TODO
 962                 model: interface model, ignored #TODO
 963                 mac_address: used for  SR-IOV ifaces #TODO for other types
 964                 use: 'data', 'bridge',  'mgmt'
 965                 type: 'virtual', 'PCI-PASSTHROUGH'('PF'), 'SR-IOV'('VF'), 'VFnotShared'
 966                 vim_id: filled/added by this function
 967                 floating_ip: True/False (or it can be None)
 968             'cloud_config': (optional) dictionary with:
 969             'key-pairs': (optional) list of strings with the public key to be inserted to the default user
 970             'users': (optional) list of users to be inserted, each item is a dict with:
 971                 'name': (mandatory) user name,
 972                 'key-pairs': (optional) list of strings with the public key to be inserted to the user
 973             'user-data': (optional) string is a text script to be passed directly to cloud-init
 974             'config-files': (optional). List of files to be transferred. Each item is a dict with:
 975                 'dest': (mandatory) string with the destination absolute path
 976                 'encoding': (optional, by default text). Can be one of:
 977                     'b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64'
 978                 'content' (mandatory): string with the content of the file
 979                 'permissions': (optional) string with file permissions, typically octal notation '0644'
 980                 'owner': (optional) file owner, string with the format 'owner:group'
 981             'boot-data-drive': boolean to indicate if user-data must be passed using a boot drive (hard disk)
 982             'disk_list': (optional) list with additional disks to the VM. Each item is a dict with:
 983                 'image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted
 984                 'size': (mandatory) string with the size of the disk in GB
 985                 'vim_id' (optional) should use this existing volume id
 986             availability_zone_index: Index of availability_zone_list to use for this this VM. None if not AV required
 987             availability_zone_list: list of availability zones given by user in the VNFD descriptor.  Ignore if
 988                 availability_zone_index is None
 989                 #TODO ip, security groups
 990         Returns a tuple with the instance identifier and created_items or raises an exception on error
 991             created_items can be None or a dictionary where this method can include key-values that will be passed to
 992             the method delete_vminstance and action_vminstance. Can be used to store created ports, volumes, etc.
 993             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
 994             as not present.
 995         """
 996         self.logger.debug("new_vminstance input: image='%s' flavor='%s' nics='%s'",image_id, flavor_id,str(net_list))
 997         try:
 998             server = None
 999             created_items = {}
1000             # metadata = {}
1001             net_list_vim = []
1002             external_network = []   # list of external networks to be connected to instance, later on used to create floating_ip
1003             no_secured_ports = []   # List of port-is with port-security disabled
1004             self._reload_connection()
1005             # metadata_vpci = {}   # For a specific neutron plugin
1006             block_device_mapping = None
1007             for net in net_list:
1008                 if not net.get("net_id"):   # skip non connected iface
1009                     continue
1010
1011                 port_dict={
1012                     "network_id": net["net_id"],
1013                     "name": net.get("name"),
1014                     "admin_state_up": True
1015                 }
1016                 if net["type"]=="virtual":
1017                     pass
1018                     # if "vpci" in net:
1019                     #     metadata_vpci[ net["net_id"] ] = [[ net["vpci"], "" ]]
1020                 elif net["type"] == "VF" or net["type"] == "SR-IOV":  # for VF
1021                     # if "vpci" in net:
1022                     #     if "VF" not in metadata_vpci:
1023                     #         metadata_vpci["VF"]=[]
1024                     #     metadata_vpci["VF"].append([ net["vpci"], "" ])
1025                     port_dict["binding:vnic_type"]="direct"
1026                     # VIO specific Changes
1027                     if self.vim_type == "VIO":
1028                         # Need to create port with port_security_enabled = False and no-security-groups
1029                         port_dict["port_security_enabled"]=False
1030                         port_dict["provider_security_groups"]=[]
1031                         port_dict["security_groups"]=[]
1032                 else:   # For PT PCI-PASSTHROUGH
1033                     # VIO specific Changes
1034                     # Current VIO release does not support port with type 'direct-physical'
1035                     # So no need to create virtual port in case of PCI-device.
1036                     # Will update port_dict code when support gets added in next VIO release
1037                     if self.vim_type == "VIO":
1038                         raise vimconn.vimconnNotSupportedException(
1039                             "Current VIO release does not support full passthrough (PT)")
1040                     # if "vpci" in net:
1041                     #     if "PF" not in metadata_vpci:
1042                     #         metadata_vpci["PF"]=[]
1043                     #     metadata_vpci["PF"].append([ net["vpci"], "" ])
1044                     port_dict["binding:vnic_type"]="direct-physical"
1045                 if not port_dict["name"]:
1046                     port_dict["name"]=name
1047                 if net.get("mac_address"):
1048                     port_dict["mac_address"]=net["mac_address"]
1049                 if net.get("ip_address"):
1050                     port_dict["fixed_ips"] = [{'ip_address': net["ip_address"]}]
1051                     # TODO add 'subnet_id': <subnet_id>
1052                 new_port = self.neutron.create_port({"port": port_dict })
1053                 created_items["port:" + str(new_port["port"]["id"])] = True
1054                 net["mac_adress"] = new_port["port"]["mac_address"]
1055                 net["vim_id"] = new_port["port"]["id"]
1056                 # if try to use a network without subnetwork, it will return a emtpy list
1057                 fixed_ips = new_port["port"].get("fixed_ips")
1058                 if fixed_ips:
1059                     net["ip"] = fixed_ips[0].get("ip_address")
1060                 else:
1061                     net["ip"] = None
1062
1063                 port = {"port-id": new_port["port"]["id"]}
1064                 if float(self.nova.api_version.get_string()) >= 2.32:
1065                     port["tag"] = new_port["port"]["name"]
1066                 net_list_vim.append(port)
1067
1068                 if net.get('floating_ip', False):
1069                     net['exit_on_floating_ip_error'] = True
1070                     external_network.append(net)
1071                 elif net['use'] == 'mgmt' and self.config.get('use_floating_ip'):
1072                     net['exit_on_floating_ip_error'] = False
1073                     external_network.append(net)
1074                     net['floating_ip'] = self.config.get('use_floating_ip')
1075
1076                 # If port security is disabled when the port has not yet been attached to the VM, then all vm traffic is dropped.
1077                 # As a workaround we wait until the VM is active and then disable the port-security
1078                 if net.get("port_security") == False and not self.config.get("no_port_security_extension"):
1079                     no_secured_ports.append(new_port["port"]["id"])
1080
1081             # if metadata_vpci:
1082             #     metadata = {"pci_assignement": json.dumps(metadata_vpci)}
1083             #     if len(metadata["pci_assignement"]) >255:
1084             #         #limit the metadata size
1085             #         #metadata["pci_assignement"] = metadata["pci_assignement"][0:255]
1086             #         self.logger.warn("Metadata deleted since it exceeds the expected length (255) ")
1087             #         metadata = {}
1088
1089             self.logger.debug("name '%s' image_id '%s'flavor_id '%s' net_list_vim '%s' description '%s'",
1090                               name, image_id, flavor_id, str(net_list_vim), description)
1091
1092             security_groups = self.config.get('security_groups')
1093             if type(security_groups) is str:
1094                 security_groups = ( security_groups, )
1095             # cloud config
1096             config_drive, userdata = self._create_user_data(cloud_config)
1097
1098             # Create additional volumes in case these are present in disk_list
1099             base_disk_index = ord('b')
1100             if disk_list:
1101                 block_device_mapping = {}
1102                 for disk in disk_list:
1103                     if disk.get('vim_id'):
1104                         block_device_mapping['_vd' + chr(base_disk_index)] = disk['vim_id']
1105                     else:
1106                         if 'image_id' in disk:
1107                             volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1108                                                                 chr(base_disk_index), imageRef=disk['image_id'])
1109                         else:
1110                             volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1111                                                                 chr(base_disk_index))
1112                         created_items["volume:" + str(volume.id)] = True
1113                         block_device_mapping['_vd' + chr(base_disk_index)] = volume.id
1114                     base_disk_index += 1
1115
1116                 # Wait until created volumes are with status available
1117                 elapsed_time = 0
1118                 while elapsed_time < volume_timeout:
1119                     for created_item in created_items:
1120                         v, _, volume_id = created_item.partition(":")
1121                         if v == 'volume':
1122                             if self.cinder.volumes.get(volume_id).status != 'available':
1123                                 break
1124                     else:  # all ready: break from while
1125                         break
1126                     time.sleep(5)
1127                     elapsed_time += 5
1128                 # If we exceeded the timeout rollback
1129                 if elapsed_time >= volume_timeout:
1130                     raise vimconn.vimconnException('Timeout creating volumes for instance ' + name,
1131                                                    http_code=vimconn.HTTP_Request_Timeout)
1132             # get availability Zone
1133             vm_av_zone = self._get_vm_availability_zone(availability_zone_index, availability_zone_list)
1134
1135             self.logger.debug("nova.servers.create({}, {}, {}, nics={}, security_groups={}, "
1136                               "availability_zone={}, key_name={}, userdata={}, config_drive={}, "
1137                               "block_device_mapping={})".format(name, image_id, flavor_id, net_list_vim,
1138                                                                 security_groups, vm_av_zone, self.config.get('keypair'),
1139                                                                 userdata, config_drive, block_device_mapping))
1140             server = self.nova.servers.create(name, image_id, flavor_id, nics=net_list_vim,
1141                                               security_groups=security_groups,
1142                                               availability_zone=vm_av_zone,
1143                                               key_name=self.config.get('keypair'),
1144                                               userdata=userdata,
1145                                               config_drive=config_drive,
1146                                               block_device_mapping=block_device_mapping
1147                                               )  # , description=description)
1148
1149             vm_start_time = time.time()
1150             # Previously mentioned workaround to wait until the VM is active and then disable the port-security
1151             if no_secured_ports:
1152                 self.__wait_for_vm(server.id, 'ACTIVE')
1153
1154             for port_id in no_secured_ports:
1155                 try:
1156                     self.neutron.update_port(port_id,
1157                                              {"port": {"port_security_enabled": False, "security_groups": None}})
1158                 except Exception as e:
1159                     raise vimconn.vimconnException("It was not possible to disable port security for port {}".format(
1160                         port_id))
1161             # print "DONE :-)", server
1162
1163             # pool_id = None
1164             if external_network:
1165                 floating_ips = self.neutron.list_floatingips().get("floatingips", ())
1166             for floating_network in external_network:
1167                 try:
1168                     assigned = False
1169                     while not assigned:
1170                         if floating_ips:
1171                             ip = floating_ips.pop(0)
1172                             if ip.get("port_id", False) or ip.get('tenant_id') != server.tenant_id:
1173                                 continue
1174                             if isinstance(floating_network['floating_ip'], str):
1175                                 if ip.get("floating_network_id") != floating_network['floating_ip']:
1176                                     continue
1177                             free_floating_ip = ip.get("floating_ip_address")
1178                         else:
1179                             if isinstance(floating_network['floating_ip'], str) and \
1180                                 floating_network['floating_ip'].lower() != "true":
1181                                 pool_id = floating_network['floating_ip']
1182                             else:
1183                                 # Find the external network
1184                                 external_nets = list()
1185                                 for net in self.neutron.list_networks()['networks']:
1186                                     if net['router:external']:
1187                                             external_nets.append(net)
1188
1189                                 if len(external_nets) == 0:
1190                                     raise vimconn.vimconnException("Cannot create floating_ip automatically since no external "
1191                                                                    "network is present",
1192                                                                     http_code=vimconn.HTTP_Conflict)
1193                                 if len(external_nets) > 1:
1194                                     raise vimconn.vimconnException("Cannot create floating_ip automatically since multiple "
1195                                                                    "external networks are present",
1196                                                                    http_code=vimconn.HTTP_Conflict)
1197
1198                                 pool_id = external_nets[0].get('id')
1199                             param = {'floatingip': {'floating_network_id': pool_id, 'tenant_id': server.tenant_id}}
1200                             try:
1201                                 # self.logger.debug("Creating floating IP")
1202                                 new_floating_ip = self.neutron.create_floatingip(param)
1203                                 free_floating_ip = new_floating_ip['floatingip']['floating_ip_address']
1204                             except Exception as e:
1205                                 raise vimconn.vimconnException(type(e).__name__ + ": Cannot create new floating_ip " +
1206                                                                str(e), http_code=vimconn.HTTP_Conflict)
1207
1208                         fix_ip = floating_network.get('ip')
1209                         while not assigned:
1210                             try:
1211                                 server.add_floating_ip(free_floating_ip, fix_ip)
1212                                 assigned = True
1213                             except Exception as e:
1214                                 # openstack need some time after VM creation to asign an IP. So retry if fails
1215                                 vm_status = self.nova.servers.get(server.id).status
1216                                 if vm_status != 'ACTIVE' and vm_status != 'ERROR':
1217                                     if time.time() - vm_start_time < server_timeout:
1218                                         time.sleep(5)
1219                                         continue
1220                                 raise vimconn.vimconnException(
1221                                     "Cannot create floating_ip: {} {}".format(type(e).__name__, e),
1222                                     http_code=vimconn.HTTP_Conflict)
1223
1224                 except Exception as e:
1225                     if not floating_network['exit_on_floating_ip_error']:
1226                         self.logger.warn("Cannot create floating_ip. %s", str(e))
1227                         continue
1228                     raise
1229
1230             return server.id, created_items
1231 #        except nvExceptions.NotFound as e:
1232 #            error_value=-vimconn.HTTP_Not_Found
1233 #            error_text= "vm instance %s not found" % vm_id
1234 #        except TypeError as e:
1235 #            raise vimconn.vimconnException(type(e).__name__ + ": "+  str(e), http_code=vimconn.HTTP_Bad_Request)
1236
1237         except Exception as e:
1238             server_id = None
1239             if server:
1240                 server_id = server.id
1241             try:
1242                 self.delete_vminstance(server_id, created_items)
1243             except Exception as e2:
1244                 self.logger.error("new_vminstance rollback fail {}".format(e2))
1245
1246             self._format_exception(e)
1247
1248     def get_vminstance(self,vm_id):
1249         '''Returns the VM instance information from VIM'''
1250         #self.logger.debug("Getting VM from VIM")
1251         try:
1252             self._reload_connection()
1253             server = self.nova.servers.find(id=vm_id)
1254             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
1255             return server.to_dict()
1256         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1257             self._format_exception(e)
1258
1259     def get_vminstance_console(self,vm_id, console_type="vnc"):
1260         '''
1261         Get a console for the virtual machine
1262         Params:
1263             vm_id: uuid of the VM
1264             console_type, can be:
1265                 "novnc" (by default), "xvpvnc" for VNC types,
1266                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
1267         Returns dict with the console parameters:
1268                 protocol: ssh, ftp, http, https, ...
1269                 server:   usually ip address
1270                 port:     the http, ssh, ... port
1271                 suffix:   extra text, e.g. the http path and query string
1272         '''
1273         self.logger.debug("Getting VM CONSOLE from VIM")
1274         try:
1275             self._reload_connection()
1276             server = self.nova.servers.find(id=vm_id)
1277             if console_type == None or console_type == "novnc":
1278                 console_dict = server.get_vnc_console("novnc")
1279             elif console_type == "xvpvnc":
1280                 console_dict = server.get_vnc_console(console_type)
1281             elif console_type == "rdp-html5":
1282                 console_dict = server.get_rdp_console(console_type)
1283             elif console_type == "spice-html5":
1284                 console_dict = server.get_spice_console(console_type)
1285             else:
1286                 raise vimconn.vimconnException("console type '{}' not allowed".format(console_type), http_code=vimconn.HTTP_Bad_Request)
1287
1288             console_dict1 = console_dict.get("console")
1289             if console_dict1:
1290                 console_url = console_dict1.get("url")
1291                 if console_url:
1292                     #parse console_url
1293                     protocol_index = console_url.find("//")
1294                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1295                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1296                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1297                         return -vimconn.HTTP_Internal_Server_Error, "Unexpected response from VIM"
1298                     console_dict={"protocol": console_url[0:protocol_index],
1299                                   "server":   console_url[protocol_index+2:port_index],
1300                                   "port":     console_url[port_index:suffix_index],
1301                                   "suffix":   console_url[suffix_index+1:]
1302                                   }
1303                     protocol_index += 2
1304                     return console_dict
1305             raise vimconn.vimconnUnexpectedResponse("Unexpected response from VIM")
1306
1307         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.BadRequest, ConnectionError) as e:
1308             self._format_exception(e)
1309
1310     def delete_vminstance(self, vm_id, created_items=None):
1311         '''Removes a VM instance from VIM. Returns the old identifier
1312         '''
1313         #print "osconnector: Getting VM from VIM"
1314         if created_items == None:
1315             created_items = {}
1316         try:
1317             self._reload_connection()
1318             # delete VM ports attached to this networks before the virtual machine
1319             for k, v in created_items.items():
1320                 if not v:  # skip already deleted
1321                     continue
1322                 try:
1323                     k_item, _, k_id = k.partition(":")
1324                     if k_item == "port":
1325                         self.neutron.delete_port(k_id)
1326                 except Exception as e:
1327                     self.logger.error("Error deleting port: {}: {}".format(type(e).__name__, e))
1328
1329             # #commented because detaching the volumes makes the servers.delete not work properly ?!?
1330             # #dettach volumes attached
1331             # server = self.nova.servers.get(vm_id)
1332             # volumes_attached_dict = server._info['os-extended-volumes:volumes_attached']   #volume['id']
1333             # #for volume in volumes_attached_dict:
1334             # #    self.cinder.volumes.detach(volume['id'])
1335
1336             if vm_id:
1337                 self.nova.servers.delete(vm_id)
1338
1339             # delete volumes. Although having detached, they should have in active status before deleting
1340             # we ensure in this loop
1341             keep_waiting = True
1342             elapsed_time = 0
1343             while keep_waiting and elapsed_time < volume_timeout:
1344                 keep_waiting = False
1345                 for k, v in created_items.items():
1346                     if not v:  # skip already deleted
1347                         continue
1348                     try:
1349                         k_item, _, k_id = k.partition(":")
1350                         if k_item == "volume":
1351                             if self.cinder.volumes.get(k_id).status != 'available':
1352                                 keep_waiting = True
1353                             else:
1354                                 self.cinder.volumes.delete(k_id)
1355                     except Exception as e:
1356                         self.logger.error("Error deleting volume: {}: {}".format(type(e).__name__, e))
1357                 if keep_waiting:
1358                     time.sleep(1)
1359                     elapsed_time += 1
1360             return None
1361         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
1362             self._format_exception(e)
1363
1364     def refresh_vms_status(self, vm_list):
1365         '''Get the status of the virtual machines and their interfaces/ports
1366            Params: the list of VM identifiers
1367            Returns a dictionary with:
1368                 vm_id:          #VIM id of this Virtual Machine
1369                     status:     #Mandatory. Text with one of:
1370                                 #  DELETED (not found at vim)
1371                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1372                                 #  OTHER (Vim reported other status not understood)
1373                                 #  ERROR (VIM indicates an ERROR status)
1374                                 #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
1375                                 #  CREATING (on building process), ERROR
1376                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
1377                                 #
1378                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
1379                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
1380                     interfaces:
1381                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
1382                         mac_address:      #Text format XX:XX:XX:XX:XX:XX
1383                         vim_net_id:       #network id where this interface is connected
1384                         vim_interface_id: #interface/port VIM id
1385                         ip_address:       #null, or text with IPv4, IPv6 address
1386                         compute_node:     #identification of compute node where PF,VF interface is allocated
1387                         pci:              #PCI address of the NIC that hosts the PF,VF
1388                         vlan:             #physical VLAN used for VF
1389         '''
1390         vm_dict={}
1391         self.logger.debug("refresh_vms status: Getting tenant VM instance information from VIM")
1392         for vm_id in vm_list:
1393             vm={}
1394             try:
1395                 vm_vim = self.get_vminstance(vm_id)
1396                 if vm_vim['status'] in vmStatus2manoFormat:
1397                     vm['status']    =  vmStatus2manoFormat[ vm_vim['status'] ]
1398                 else:
1399                     vm['status']    = "OTHER"
1400                     vm['error_msg'] = "VIM status reported " + vm_vim['status']
1401                 try:
1402                     vm['vim_info']  = yaml.safe_dump(vm_vim, default_flow_style=True, width=256)
1403                 except yaml.representer.RepresenterError:
1404                     vm['vim_info'] = str(vm_vim)
1405                 vm["interfaces"] = []
1406                 if vm_vim.get('fault'):
1407                     vm['error_msg'] = str(vm_vim['fault'])
1408                 #get interfaces
1409                 try:
1410                     self._reload_connection()
1411                     port_dict = self.neutron.list_ports(device_id=vm_id)
1412                     for port in port_dict["ports"]:
1413                         interface={}
1414                         try:
1415                             interface['vim_info'] = yaml.safe_dump(port, default_flow_style=True, width=256)
1416                         except yaml.representer.RepresenterError:
1417                             interface['vim_info'] = str(port)
1418                         interface["mac_address"] = port.get("mac_address")
1419                         interface["vim_net_id"] = port["network_id"]
1420                         interface["vim_interface_id"] = port["id"]
1421                         # check if OS-EXT-SRV-ATTR:host is there,
1422                         # in case of non-admin credentials, it will be missing
1423                         if vm_vim.get('OS-EXT-SRV-ATTR:host'):
1424                             interface["compute_node"] = vm_vim['OS-EXT-SRV-ATTR:host']
1425                         interface["pci"] = None
1426
1427                         # check if binding:profile is there,
1428                         # in case of non-admin credentials, it will be missing
1429                         if port.get('binding:profile'):
1430                             if port['binding:profile'].get('pci_slot'):
1431                                 # TODO: At the moment sr-iov pci addresses are converted to PF pci addresses by setting the slot to 0x00
1432                                 # TODO: This is just a workaround valid for niantinc. Find a better way to do so
1433                                 #   CHANGE DDDD:BB:SS.F to DDDD:BB:00.(F%2)   assuming there are 2 ports per nic
1434                                 pci = port['binding:profile']['pci_slot']
1435                                 # interface["pci"] = pci[:-4] + "00." + str(int(pci[-1]) % 2)
1436                                 interface["pci"] = pci
1437                         interface["vlan"] = None
1438                         #if network is of type vlan and port is of type direct (sr-iov) then set vlan id
1439                         network = self.neutron.show_network(port["network_id"])
1440                         if network['network'].get('provider:network_type') == 'vlan' and \
1441                             port.get("binding:vnic_type") == "direct":
1442                             interface["vlan"] = network['network'].get('provider:segmentation_id')
1443                         ips=[]
1444                         #look for floating ip address
1445                         try:
1446                             floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
1447                             if floating_ip_dict.get("floatingips"):
1448                                 ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address") )
1449                         except Exception:
1450                             pass
1451
1452                         for subnet in port["fixed_ips"]:
1453                             ips.append(subnet["ip_address"])
1454                         interface["ip_address"] = ";".join(ips)
1455                         vm["interfaces"].append(interface)
1456                 except Exception as e:
1457                     self.logger.error("Error getting vm interface information {}: {}".format(type(e).__name__, e),
1458                                       exc_info=True)
1459             except vimconn.vimconnNotFoundException as e:
1460                 self.logger.error("Exception getting vm status: %s", str(e))
1461                 vm['status'] = "DELETED"
1462                 vm['error_msg'] = str(e)
1463             except vimconn.vimconnException as e:
1464                 self.logger.error("Exception getting vm status: %s", str(e))
1465                 vm['status'] = "VIM_ERROR"
1466                 vm['error_msg'] = str(e)
1467             vm_dict[vm_id] = vm
1468         return vm_dict
1469
1470     def action_vminstance(self, vm_id, action_dict, created_items={}):
1471         '''Send and action over a VM instance from VIM
1472         Returns None or the console dict if the action was successfully sent to the VIM'''
1473         self.logger.debug("Action over VM '%s': %s", vm_id, str(action_dict))
1474         try:
1475             self._reload_connection()
1476             server = self.nova.servers.find(id=vm_id)
1477             if "start" in action_dict:
1478                 if action_dict["start"]=="rebuild":
1479                     server.rebuild()
1480                 else:
1481                     if server.status=="PAUSED":
1482                         server.unpause()
1483                     elif server.status=="SUSPENDED":
1484                         server.resume()
1485                     elif server.status=="SHUTOFF":
1486                         server.start()
1487             elif "pause" in action_dict:
1488                 server.pause()
1489             elif "resume" in action_dict:
1490                 server.resume()
1491             elif "shutoff" in action_dict or "shutdown" in action_dict:
1492                 server.stop()
1493             elif "forceOff" in action_dict:
1494                 server.stop() #TODO
1495             elif "terminate" in action_dict:
1496                 server.delete()
1497             elif "createImage" in action_dict:
1498                 server.create_image()
1499                 #"path":path_schema,
1500                 #"description":description_schema,
1501                 #"name":name_schema,
1502                 #"metadata":metadata_schema,
1503                 #"imageRef": id_schema,
1504                 #"disk": {"oneOf":[{"type": "null"}, {"type":"string"}] },
1505             elif "rebuild" in action_dict:
1506                 server.rebuild(server.image['id'])
1507             elif "reboot" in action_dict:
1508                 server.reboot() #reboot_type='SOFT'
1509             elif "console" in action_dict:
1510                 console_type = action_dict["console"]
1511                 if console_type == None or console_type == "novnc":
1512                     console_dict = server.get_vnc_console("novnc")
1513                 elif console_type == "xvpvnc":
1514                     console_dict = server.get_vnc_console(console_type)
1515                 elif console_type == "rdp-html5":
1516                     console_dict = server.get_rdp_console(console_type)
1517                 elif console_type == "spice-html5":
1518                     console_dict = server.get_spice_console(console_type)
1519                 else:
1520                     raise vimconn.vimconnException("console type '{}' not allowed".format(console_type),
1521                                                    http_code=vimconn.HTTP_Bad_Request)
1522                 try:
1523                     console_url = console_dict["console"]["url"]
1524                     #parse console_url
1525                     protocol_index = console_url.find("//")
1526                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1527                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1528                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1529                         raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1530                     console_dict2={"protocol": console_url[0:protocol_index],
1531                                   "server":   console_url[protocol_index+2 : port_index],
1532                                   "port":     int(console_url[port_index+1 : suffix_index]),
1533                                   "suffix":   console_url[suffix_index+1:]
1534                                   }
1535                     return console_dict2
1536                 except Exception as e:
1537                     raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1538
1539             return None
1540         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1541             self._format_exception(e)
1542         #TODO insert exception vimconn.HTTP_Unauthorized
1543
1544     ####### VIO Specific Changes #########
1545     def _genrate_vlanID(self):
1546         """
1547          Method to get unused vlanID
1548             Args:
1549                 None
1550             Returns:
1551                 vlanID
1552         """
1553         #Get used VLAN IDs
1554         usedVlanIDs = []
1555         networks = self.get_network_list()
1556         for net in networks:
1557             if net.get('provider:segmentation_id'):
1558                 usedVlanIDs.append(net.get('provider:segmentation_id'))
1559         used_vlanIDs = set(usedVlanIDs)
1560
1561         #find unused VLAN ID
1562         for vlanID_range in self.config.get('dataplane_net_vlan_range'):
1563             try:
1564                 start_vlanid , end_vlanid = map(int, vlanID_range.replace(" ", "").split("-"))
1565                 for vlanID in xrange(start_vlanid, end_vlanid + 1):
1566                     if vlanID not in used_vlanIDs:
1567                         return vlanID
1568             except Exception as exp:
1569                 raise vimconn.vimconnException("Exception {} occurred while generating VLAN ID.".format(exp))
1570         else:
1571             raise vimconn.vimconnConflictException("Unable to create the SRIOV VLAN network."\
1572                 " All given Vlan IDs {} are in use.".format(self.config.get('dataplane_net_vlan_range')))
1573
1574
1575     def _validate_vlan_ranges(self, dataplane_net_vlan_range):
1576         """
1577         Method to validate user given vlanID ranges
1578             Args:  None
1579             Returns: None
1580         """
1581         for vlanID_range in dataplane_net_vlan_range:
1582             vlan_range = vlanID_range.replace(" ", "")
1583             #validate format
1584             vlanID_pattern = r'(\d)*-(\d)*$'
1585             match_obj = re.match(vlanID_pattern, vlan_range)
1586             if not match_obj:
1587                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}.You must provide "\
1588                 "'dataplane_net_vlan_range' in format [start_ID - end_ID].".format(vlanID_range))
1589
1590             start_vlanid , end_vlanid = map(int,vlan_range.split("-"))
1591             if start_vlanid <= 0 :
1592                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1593                 "Start ID can not be zero. For VLAN "\
1594                 "networks valid IDs are 1 to 4094 ".format(vlanID_range))
1595             if end_vlanid > 4094 :
1596                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1597                 "End VLAN ID can not be greater than 4094. For VLAN "\
1598                 "networks valid IDs are 1 to 4094 ".format(vlanID_range))
1599
1600             if start_vlanid > end_vlanid:
1601                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1602                     "You must provide a 'dataplane_net_vlan_range' in format start_ID - end_ID and "\
1603                     "start_ID < end_ID ".format(vlanID_range))
1604
1605 #NOT USED FUNCTIONS
1606
1607     def new_external_port(self, port_data):
1608         #TODO openstack if needed
1609         '''Adds a external port to VIM'''
1610         '''Returns the port identifier'''
1611         return -vimconn.HTTP_Internal_Server_Error, "osconnector.new_external_port() not implemented"
1612
1613     def connect_port_network(self, port_id, network_id, admin=False):
1614         #TODO openstack if needed
1615         '''Connects a external port to a network'''
1616         '''Returns status code of the VIM response'''
1617         return -vimconn.HTTP_Internal_Server_Error, "osconnector.connect_port_network() not implemented"
1618
1619     def new_user(self, user_name, user_passwd, tenant_id=None):
1620         '''Adds a new user to openstack VIM'''
1621         '''Returns the user identifier'''
1622         self.logger.debug("osconnector: Adding a new user to VIM")
1623         try:
1624             self._reload_connection()
1625             user=self.keystone.users.create(user_name, password=user_passwd, default_project=tenant_id)
1626             #self.keystone.tenants.add_user(self.k_creds["username"], #role)
1627             return user.id
1628         except ksExceptions.ConnectionError as e:
1629             error_value=-vimconn.HTTP_Bad_Request
1630             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1631         except ksExceptions.ClientException as e: #TODO remove
1632             error_value=-vimconn.HTTP_Bad_Request
1633             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1634         #TODO insert exception vimconn.HTTP_Unauthorized
1635         #if reaching here is because an exception
1636         self.logger.debug("new_user " + error_text)
1637         return error_value, error_text
1638
1639     def delete_user(self, user_id):
1640         '''Delete a user from openstack VIM'''
1641         '''Returns the user identifier'''
1642         if self.debug:
1643             print("osconnector: Deleting  a  user from VIM")
1644         try:
1645             self._reload_connection()
1646             self.keystone.users.delete(user_id)
1647             return 1, user_id
1648         except ksExceptions.ConnectionError as e:
1649             error_value=-vimconn.HTTP_Bad_Request
1650             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1651         except ksExceptions.NotFound as e:
1652             error_value=-vimconn.HTTP_Not_Found
1653             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1654         except ksExceptions.ClientException as e: #TODO remove
1655             error_value=-vimconn.HTTP_Bad_Request
1656             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1657         #TODO insert exception vimconn.HTTP_Unauthorized
1658         #if reaching here is because an exception
1659             self.logger.debug("delete_tenant " + error_text)
1660         return error_value, error_text
1661
1662     def get_hosts_info(self):
1663         '''Get the information of deployed hosts
1664         Returns the hosts content'''
1665         if self.debug:
1666             print("osconnector: Getting Host info from VIM")
1667         try:
1668             h_list=[]
1669             self._reload_connection()
1670             hypervisors = self.nova.hypervisors.list()
1671             for hype in hypervisors:
1672                 h_list.append( hype.to_dict() )
1673             return 1, {"hosts":h_list}
1674         except nvExceptions.NotFound as e:
1675             error_value=-vimconn.HTTP_Not_Found
1676             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1677         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1678             error_value=-vimconn.HTTP_Bad_Request
1679             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1680         #TODO insert exception vimconn.HTTP_Unauthorized
1681         #if reaching here is because an exception
1682         self.logger.debug("get_hosts_info " + error_text)
1683         return error_value, error_text
1684
1685     def get_hosts(self, vim_tenant):
1686         '''Get the hosts and deployed instances
1687         Returns the hosts content'''
1688         r, hype_dict = self.get_hosts_info()
1689         if r<0:
1690             return r, hype_dict
1691         hypervisors = hype_dict["hosts"]
1692         try:
1693             servers = self.nova.servers.list()
1694             for hype in hypervisors:
1695                 for server in servers:
1696                     if server.to_dict()['OS-EXT-SRV-ATTR:hypervisor_hostname']==hype['hypervisor_hostname']:
1697                         if 'vm' in hype:
1698                             hype['vm'].append(server.id)
1699                         else:
1700                             hype['vm'] = [server.id]
1701             return 1, hype_dict
1702         except nvExceptions.NotFound as e:
1703             error_value=-vimconn.HTTP_Not_Found
1704             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1705         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1706             error_value=-vimconn.HTTP_Bad_Request
1707             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1708         #TODO insert exception vimconn.HTTP_Unauthorized
1709         #if reaching here is because an exception
1710         self.logger.debug("get_hosts " + error_text)
1711         return error_value, error_text
1712
1713     def new_classification(self, name, ctype, definition):
1714         self.logger.debug(
1715             'Adding a new (Traffic) Classification to VIM, named %s', name)
1716         try:
1717             new_class = None
1718             self._reload_connection()
1719             if ctype not in supportedClassificationTypes:
1720                 raise vimconn.vimconnNotSupportedException(
1721                         'OpenStack VIM connector doesn\'t support provided '
1722                         'Classification Type {}, supported ones are: '
1723                         '{}'.format(ctype, supportedClassificationTypes))
1724             if not self._validate_classification(ctype, definition):
1725                 raise vimconn.vimconnException(
1726                     'Incorrect Classification definition '
1727                     'for the type specified.')
1728             classification_dict = definition
1729             classification_dict['name'] = name
1730
1731             new_class = self.neutron.create_sfc_flow_classifier(
1732                 {'flow_classifier': classification_dict})
1733             return new_class['flow_classifier']['id']
1734         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1735                 neExceptions.NeutronException, ConnectionError) as e:
1736             self.logger.error(
1737                 'Creation of Classification failed.')
1738             self._format_exception(e)
1739
1740     def get_classification(self, class_id):
1741         self.logger.debug(" Getting Classification %s from VIM", class_id)
1742         filter_dict = {"id": class_id}
1743         class_list = self.get_classification_list(filter_dict)
1744         if len(class_list) == 0:
1745             raise vimconn.vimconnNotFoundException(
1746                 "Classification '{}' not found".format(class_id))
1747         elif len(class_list) > 1:
1748             raise vimconn.vimconnConflictException(
1749                 "Found more than one Classification with this criteria")
1750         classification = class_list[0]
1751         return classification
1752
1753     def get_classification_list(self, filter_dict={}):
1754         self.logger.debug("Getting Classifications from VIM filter: '%s'",
1755                           str(filter_dict))
1756         try:
1757             filter_dict_os = filter_dict.copy()
1758             self._reload_connection()
1759             if self.api_version3 and "tenant_id" in filter_dict_os:
1760                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1761             classification_dict = self.neutron.list_sfc_flow_classifiers(
1762                 **filter_dict_os)
1763             classification_list = classification_dict["flow_classifiers"]
1764             self.__classification_os2mano(classification_list)
1765             return classification_list
1766         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1767                 neExceptions.NeutronException, ConnectionError) as e:
1768             self._format_exception(e)
1769
1770     def delete_classification(self, class_id):
1771         self.logger.debug("Deleting Classification '%s' from VIM", class_id)
1772         try:
1773             self._reload_connection()
1774             self.neutron.delete_sfc_flow_classifier(class_id)
1775             return class_id
1776         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1777                 ksExceptions.ClientException, neExceptions.NeutronException,
1778                 ConnectionError) as e:
1779             self._format_exception(e)
1780
1781     def new_sfi(self, name, ingress_ports, egress_ports, sfc_encap=True):
1782         self.logger.debug(
1783             "Adding a new Service Function Instance to VIM, named '%s'", name)
1784         try:
1785             new_sfi = None
1786             self._reload_connection()
1787             correlation = None
1788             if sfc_encap:
1789                 correlation = 'nsh'
1790             if len(ingress_ports) != 1:
1791                 raise vimconn.vimconnNotSupportedException(
1792                     "OpenStack VIM connector can only have "
1793                     "1 ingress port per SFI")
1794             if len(egress_ports) != 1:
1795                 raise vimconn.vimconnNotSupportedException(
1796                     "OpenStack VIM connector can only have "
1797                     "1 egress port per SFI")
1798             sfi_dict = {'name': name,
1799                         'ingress': ingress_ports[0],
1800                         'egress': egress_ports[0],
1801                         'service_function_parameters': {
1802                             'correlation': correlation}}
1803             new_sfi = self.neutron.create_sfc_port_pair({'port_pair': sfi_dict})
1804             return new_sfi['port_pair']['id']
1805         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1806                 neExceptions.NeutronException, ConnectionError) as e:
1807             if new_sfi:
1808                 try:
1809                     self.neutron.delete_sfc_port_pair(
1810                         new_sfi['port_pair']['id'])
1811                 except Exception:
1812                     self.logger.error(
1813                         'Creation of Service Function Instance failed, with '
1814                         'subsequent deletion failure as well.')
1815             self._format_exception(e)
1816
1817     def get_sfi(self, sfi_id):
1818         self.logger.debug(
1819             'Getting Service Function Instance %s from VIM', sfi_id)
1820         filter_dict = {"id": sfi_id}
1821         sfi_list = self.get_sfi_list(filter_dict)
1822         if len(sfi_list) == 0:
1823             raise vimconn.vimconnNotFoundException(
1824                 "Service Function Instance '{}' not found".format(sfi_id))
1825         elif len(sfi_list) > 1:
1826             raise vimconn.vimconnConflictException(
1827                 'Found more than one Service Function Instance '
1828                 'with this criteria')
1829         sfi = sfi_list[0]
1830         return sfi
1831
1832     def get_sfi_list(self, filter_dict={}):
1833         self.logger.debug("Getting Service Function Instances from "
1834                           "VIM filter: '%s'", str(filter_dict))
1835         try:
1836             self._reload_connection()
1837             filter_dict_os = filter_dict.copy()
1838             if self.api_version3 and "tenant_id" in filter_dict_os:
1839                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1840             sfi_dict = self.neutron.list_sfc_port_pairs(**filter_dict_os)
1841             sfi_list = sfi_dict["port_pairs"]
1842             self.__sfi_os2mano(sfi_list)
1843             return sfi_list
1844         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1845                 neExceptions.NeutronException, ConnectionError) as e:
1846             self._format_exception(e)
1847
1848     def delete_sfi(self, sfi_id):
1849         self.logger.debug("Deleting Service Function Instance '%s' "
1850                           "from VIM", sfi_id)
1851         try:
1852             self._reload_connection()
1853             self.neutron.delete_sfc_port_pair(sfi_id)
1854             return sfi_id
1855         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1856                 ksExceptions.ClientException, neExceptions.NeutronException,
1857                 ConnectionError) as e:
1858             self._format_exception(e)
1859
1860     def new_sf(self, name, sfis, sfc_encap=True):
1861         self.logger.debug("Adding a new Service Function to VIM, "
1862                           "named '%s'", name)
1863         try:
1864             new_sf = None
1865             self._reload_connection()
1866             # correlation = None
1867             # if sfc_encap:
1868             #     correlation = 'nsh'
1869             for instance in sfis:
1870                 sfi = self.get_sfi(instance)
1871                 if sfi.get('sfc_encap') != sfc_encap:
1872                     raise vimconn.vimconnNotSupportedException(
1873                         "OpenStack VIM connector requires all SFIs of the "
1874                         "same SF to share the same SFC Encapsulation")
1875             sf_dict = {'name': name,
1876                        'port_pairs': sfis}
1877             new_sf = self.neutron.create_sfc_port_pair_group({
1878                 'port_pair_group': sf_dict})
1879             return new_sf['port_pair_group']['id']
1880         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1881                 neExceptions.NeutronException, ConnectionError) as e:
1882             if new_sf:
1883                 try:
1884                     self.neutron.delete_sfc_port_pair_group(
1885                         new_sf['port_pair_group']['id'])
1886                 except Exception:
1887                     self.logger.error(
1888                         'Creation of Service Function failed, with '
1889                         'subsequent deletion failure as well.')
1890             self._format_exception(e)
1891
1892     def get_sf(self, sf_id):
1893         self.logger.debug("Getting Service Function %s from VIM", sf_id)
1894         filter_dict = {"id": sf_id}
1895         sf_list = self.get_sf_list(filter_dict)
1896         if len(sf_list) == 0:
1897             raise vimconn.vimconnNotFoundException(
1898                 "Service Function '{}' not found".format(sf_id))
1899         elif len(sf_list) > 1:
1900             raise vimconn.vimconnConflictException(
1901                 "Found more than one Service Function with this criteria")
1902         sf = sf_list[0]
1903         return sf
1904
1905     def get_sf_list(self, filter_dict={}):
1906         self.logger.debug("Getting Service Function from VIM filter: '%s'",
1907                           str(filter_dict))
1908         try:
1909             self._reload_connection()
1910             filter_dict_os = filter_dict.copy()
1911             if self.api_version3 and "tenant_id" in filter_dict_os:
1912                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1913             sf_dict = self.neutron.list_sfc_port_pair_groups(**filter_dict_os)
1914             sf_list = sf_dict["port_pair_groups"]
1915             self.__sf_os2mano(sf_list)
1916             return sf_list
1917         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1918                 neExceptions.NeutronException, ConnectionError) as e:
1919             self._format_exception(e)
1920
1921     def delete_sf(self, sf_id):
1922         self.logger.debug("Deleting Service Function '%s' from VIM", sf_id)
1923         try:
1924             self._reload_connection()
1925             self.neutron.delete_sfc_port_pair_group(sf_id)
1926             return sf_id
1927         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1928                 ksExceptions.ClientException, neExceptions.NeutronException,
1929                 ConnectionError) as e:
1930             self._format_exception(e)
1931
1932     def new_sfp(self, name, classifications, sfs, sfc_encap=True, spi=None):
1933         self.logger.debug("Adding a new Service Function Path to VIM, "
1934                           "named '%s'", name)
1935         try:
1936             new_sfp = None
1937             self._reload_connection()
1938             # In networking-sfc the MPLS encapsulation is legacy
1939             # should be used when no full SFC Encapsulation is intended
1940             sfc_encap = 'mpls'
1941             if sfc_encap:
1942                 correlation = 'nsh'
1943             sfp_dict = {'name': name,
1944                         'flow_classifiers': classifications,
1945                         'port_pair_groups': sfs,
1946                         'chain_parameters': {'correlation': correlation}}
1947             if spi:
1948                 sfp_dict['chain_id'] = spi
1949             new_sfp = self.neutron.create_sfc_port_chain({'port_chain': sfp_dict})
1950             return new_sfp["port_chain"]["id"]
1951         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1952                 neExceptions.NeutronException, ConnectionError) as e:
1953             if new_sfp:
1954                 try:
1955                     self.neutron.delete_sfc_port_chain(new_sfp['port_chain']['id'])
1956                 except Exception:
1957                     self.logger.error(
1958                         'Creation of Service Function Path failed, with '
1959                         'subsequent deletion failure as well.')
1960             self._format_exception(e)
1961
1962     def get_sfp(self, sfp_id):
1963         self.logger.debug(" Getting Service Function Path %s from VIM", sfp_id)
1964         filter_dict = {"id": sfp_id}
1965         sfp_list = self.get_sfp_list(filter_dict)
1966         if len(sfp_list) == 0:
1967             raise vimconn.vimconnNotFoundException(
1968                 "Service Function Path '{}' not found".format(sfp_id))
1969         elif len(sfp_list) > 1:
1970             raise vimconn.vimconnConflictException(
1971                 "Found more than one Service Function Path with this criteria")
1972         sfp = sfp_list[0]
1973         return sfp
1974
1975     def get_sfp_list(self, filter_dict={}):
1976         self.logger.debug("Getting Service Function Paths from VIM filter: "
1977                           "'%s'", str(filter_dict))
1978         try:
1979             self._reload_connection()
1980             filter_dict_os = filter_dict.copy()
1981             if self.api_version3 and "tenant_id" in filter_dict_os:
1982                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1983             sfp_dict = self.neutron.list_sfc_port_chains(**filter_dict_os)
1984             sfp_list = sfp_dict["port_chains"]
1985             self.__sfp_os2mano(sfp_list)
1986             return sfp_list
1987         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1988                 neExceptions.NeutronException, ConnectionError) as e:
1989             self._format_exception(e)
1990
1991     def delete_sfp(self, sfp_id):
1992         self.logger.debug(
1993             "Deleting Service Function Path '%s' from VIM", sfp_id)
1994         try:
1995             self._reload_connection()
1996             self.neutron.delete_sfc_port_chain(sfp_id)
1997             return sfp_id
1998         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1999                 ksExceptions.ClientException, neExceptions.NeutronException,
2000                 ConnectionError) as e:
2001             self._format_exception(e)