osm_ro/vimconn_openstack.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 osconnector implements all the methods to interact with openstack using the python-neutronclient.
  26
  27 For the VNF forwarding graph, The OpenStack VIM connector calls the
  28 networking-sfc Neutron extension methods, whose resources are mapped
  29 to the VIM connector's SFC resources as follows:
  30 - Classification (OSM) -> Flow Classifier (Neutron)
  31 - Service Function Instance (OSM) -> Port Pair (Neutron)
  32 - Service Function (OSM) -> Port Pair Group (Neutron)
  33 - Service Function Path (OSM) -> Port Chain (Neutron)
  34 '''
  35 __author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes, xFlow Research, Igor D.C."
  36 __date__  = "$22-sep-2017 23:59:59$"
  37
  38 import vimconn
  39 # import json
  40 import logging
  41 import netaddr
  42 import time
  43 import yaml
  44 import random
  45 import re
  46 import copy
  47
  48 from novaclient import client as nClient, exceptions as nvExceptions
  49 from keystoneauth1.identity import v2, v3
  50 from keystoneauth1 import session
  51 import keystoneclient.exceptions as ksExceptions
  52 import keystoneclient.v3.client as ksClient_v3
  53 import keystoneclient.v2_0.client as ksClient_v2
  54 from glanceclient import client as glClient
  55 import glanceclient.client as gl1Client
  56 import glanceclient.exc as gl1Exceptions
  57 from  cinderclient import client as cClient
  58 from httplib import HTTPException
  59 from neutronclient.neutron import client as neClient
  60 from neutronclient.common import exceptions as neExceptions
  61 from requests.exceptions import ConnectionError
  62
  63
  64 """contain the openstack virtual machine status to openmano status"""
  65 vmStatus2manoFormat={'ACTIVE':'ACTIVE',
  66                      'PAUSED':'PAUSED',
  67                      'SUSPENDED': 'SUSPENDED',
  68                      'SHUTOFF':'INACTIVE',
  69                      'BUILD':'BUILD',
  70                      'ERROR':'ERROR','DELETED':'DELETED'
  71                      }
  72 netStatus2manoFormat={'ACTIVE':'ACTIVE','PAUSED':'PAUSED','INACTIVE':'INACTIVE','BUILD':'BUILD','ERROR':'ERROR','DELETED':'DELETED'
  73                      }
  74
  75 supportedClassificationTypes = ['legacy_flow_classifier']
  76
  77 #global var to have a timeout creating and deleting volumes
  78 volume_timeout = 600
  79 server_timeout = 600
  80
  81 class vimconnector(vimconn.vimconnector):
  82     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None,
  83                  log_level=None, config={}, persistent_info={}):
  84         '''using common constructor parameters. In this case
  85         'url' is the keystone authorization url,
  86         'url_admin' is not use
  87         '''
  88         api_version = config.get('APIversion')
  89         if api_version and api_version not in ('v3.3', 'v2.0', '2', '3'):
  90             raise vimconn.vimconnException("Invalid value '{}' for config:APIversion. "
  91                                            "Allowed values are 'v3.3', 'v2.0', '2' or '3'".format(api_version))
  92         vim_type = config.get('vim_type')
  93         if vim_type and vim_type not in ('vio', 'VIO'):
  94             raise vimconn.vimconnException("Invalid value '{}' for config:vim_type."
  95                             "Allowed values are 'vio' or 'VIO'".format(vim_type))
  96
  97         if config.get('dataplane_net_vlan_range') is not None:
  98             #validate vlan ranges provided by user
  99             self._validate_vlan_ranges(config.get('dataplane_net_vlan_range'))
 100
 101         vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level,
 102                                       config)
 103
 104         self.insecure = self.config.get("insecure", False)
 105         if not url:
 106             raise TypeError('url param can not be NoneType')
 107         self.persistent_info = persistent_info
 108         self.availability_zone = persistent_info.get('availability_zone', None)
 109         self.session = persistent_info.get('session', {'reload_client': True})
 110         self.nova = self.session.get('nova')
 111         self.neutron = self.session.get('neutron')
 112         self.cinder = self.session.get('cinder')
 113         self.glance = self.session.get('glance')
 114         self.glancev1 = self.session.get('glancev1')
 115         self.keystone = self.session.get('keystone')
 116         self.api_version3 = self.session.get('api_version3')
 117         self.vim_type = self.config.get("vim_type")
 118         if self.vim_type:
 119             self.vim_type = self.vim_type.upper()
 120         if self.config.get("use_internal_endpoint"):
 121             self.endpoint_type = "internalURL"
 122         else:
 123             self.endpoint_type = None
 124
 125         self.logger = logging.getLogger('openmano.vim.openstack')
 126
 127         ####### VIO Specific Changes #########
 128         if self.vim_type == "VIO":
 129             self.logger = logging.getLogger('openmano.vim.vio')
 130
 131         if log_level:
 132             self.logger.setLevel( getattr(logging, log_level))
 133
 134     def __getitem__(self, index):
 135         """Get individuals parameters.
 136         Throw KeyError"""
 137         if index == 'project_domain_id':
 138             return self.config.get("project_domain_id")
 139         elif index == 'user_domain_id':
 140             return self.config.get("user_domain_id")
 141         else:
 142             return vimconn.vimconnector.__getitem__(self, index)
 143
 144     def __setitem__(self, index, value):
 145         """Set individuals parameters and it is marked as dirty so to force connection reload.
 146         Throw KeyError"""
 147         if index == 'project_domain_id':
 148             self.config["project_domain_id"] = value
 149         elif index == 'user_domain_id':
 150                 self.config["user_domain_id"] = value
 151         else:
 152             vimconn.vimconnector.__setitem__(self, index, value)
 153         self.session['reload_client'] = True
 154
 155     def _reload_connection(self):
 156         '''Called before any operation, it check if credentials has changed
 157         Throw keystoneclient.apiclient.exceptions.AuthorizationFailure
 158         '''
 159         #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-)
 160         if self.session['reload_client']:
 161             if self.config.get('APIversion'):
 162                 self.api_version3 = self.config['APIversion'] == 'v3.3' or self.config['APIversion'] == '3'
 163             else:  # get from ending auth_url that end with v3 or with v2.0
 164                 self.api_version3 =  self.url.endswith("/v3") or self.url.endswith("/v3/")
 165             self.session['api_version3'] = self.api_version3
 166             if self.api_version3:
 167                 if self.config.get('project_domain_id') or self.config.get('project_domain_name'):
 168                     project_domain_id_default = None
 169                 else:
 170                     project_domain_id_default = 'default'
 171                 if self.config.get('user_domain_id') or self.config.get('user_domain_name'):
 172                     user_domain_id_default = None
 173                 else:
 174                     user_domain_id_default = 'default'
 175                 auth = v3.Password(auth_url=self.url,
 176                                    username=self.user,
 177                                    password=self.passwd,
 178                                    project_name=self.tenant_name,
 179                                    project_id=self.tenant_id,
 180                                    project_domain_id=self.config.get('project_domain_id', project_domain_id_default),
 181                                    user_domain_id=self.config.get('user_domain_id', user_domain_id_default),
 182                                    project_domain_name=self.config.get('project_domain_name'),
 183                                    user_domain_name=self.config.get('user_domain_name'))
 184             else:
 185                 auth = v2.Password(auth_url=self.url,
 186                                    username=self.user,
 187                                    password=self.passwd,
 188                                    tenant_name=self.tenant_name,
 189                                    tenant_id=self.tenant_id)
 190             sess = session.Session(auth=auth, verify=not self.insecure)
 191             if self.api_version3:
 192                 self.keystone = ksClient_v3.Client(session=sess, endpoint_type=self.endpoint_type)
 193             else:
 194                 self.keystone = ksClient_v2.Client(session=sess, endpoint_type=self.endpoint_type)
 195             self.session['keystone'] = self.keystone
 196             # In order to enable microversion functionality an explicit microversion must be specified in 'config'.
 197             # This implementation approach is due to the warning message in
 198             # https://developer.openstack.org/api-guide/compute/microversions.html
 199             # where it is stated that microversion backwards compatibility is not guaranteed and clients should
 200             # always require an specific microversion.
 201             # To be able to use 'device role tagging' functionality define 'microversion: 2.32' in datacenter config
 202             version = self.config.get("microversion")
 203             if not version:
 204                 version = "2.1"
 205             self.nova = self.session['nova'] = nClient.Client(str(version), session=sess, endpoint_type=self.endpoint_type)
 206             self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess, endpoint_type=self.endpoint_type)
 207             self.cinder = self.session['cinder'] = cClient.Client(2, session=sess, endpoint_type=self.endpoint_type)
 208             if self.endpoint_type == "internalURL":
 209                 glance_service_id = self.keystone.services.list(name="glance")[0].id
 210                 glance_endpoint = self.keystone.endpoints.list(glance_service_id, interface="internal")[0].url
 211             else:
 212                 glance_endpoint = None
 213             self.glance = self.session['glance'] = glClient.Client(2, session=sess, endpoint=glance_endpoint)
 214             #using version 1 of glance client in new_image()
 215             self.glancev1 = self.session['glancev1'] = glClient.Client('1', session=sess,
 216                                                                        endpoint=glance_endpoint)
 217             self.session['reload_client'] = False
 218             self.persistent_info['session'] = self.session
 219             # add availablity zone info inside  self.persistent_info
 220             self._set_availablity_zones()
 221             self.persistent_info['availability_zone'] = self.availability_zone
 222
 223     def __net_os2mano(self, net_list_dict):
 224         '''Transform the net openstack format to mano format
 225         net_list_dict can be a list of dict or a single dict'''
 226         if type(net_list_dict) is dict:
 227             net_list_=(net_list_dict,)
 228         elif type(net_list_dict) is list:
 229             net_list_=net_list_dict
 230         else:
 231             raise TypeError("param net_list_dict must be a list or a dictionary")
 232         for net in net_list_:
 233             if net.get('provider:network_type') == "vlan":
 234                 net['type']='data'
 235             else:
 236                 net['type']='bridge'
 237
 238     def __classification_os2mano(self, class_list_dict):
 239         """Transform the openstack format (Flow Classifier) to mano format
 240         (Classification) class_list_dict can be a list of dict or a single dict
 241         """
 242         if isinstance(class_list_dict, dict):
 243             class_list_ = [class_list_dict]
 244         elif isinstance(class_list_dict, list):
 245             class_list_ = class_list_dict
 246         else:
 247             raise TypeError(
 248                 "param class_list_dict must be a list or a dictionary")
 249         for classification in class_list_:
 250             id = classification.pop('id')
 251             name = classification.pop('name')
 252             description = classification.pop('description')
 253             project_id = classification.pop('project_id')
 254             tenant_id = classification.pop('tenant_id')
 255             original_classification = copy.deepcopy(classification)
 256             classification.clear()
 257             classification['ctype'] = 'legacy_flow_classifier'
 258             classification['definition'] = original_classification
 259             classification['id'] = id
 260             classification['name'] = name
 261             classification['description'] = description
 262             classification['project_id'] = project_id
 263             classification['tenant_id'] = tenant_id
 264
 265     def __sfi_os2mano(self, sfi_list_dict):
 266         """Transform the openstack format (Port Pair) to mano format (SFI)
 267         sfi_list_dict can be a list of dict or a single dict
 268         """
 269         if isinstance(sfi_list_dict, dict):
 270             sfi_list_ = [sfi_list_dict]
 271         elif isinstance(sfi_list_dict, list):
 272             sfi_list_ = sfi_list_dict
 273         else:
 274             raise TypeError(
 275                 "param sfi_list_dict must be a list or a dictionary")
 276         for sfi in sfi_list_:
 277             sfi['ingress_ports'] = []
 278             sfi['egress_ports'] = []
 279             if sfi.get('ingress'):
 280                 sfi['ingress_ports'].append(sfi['ingress'])
 281             if sfi.get('egress'):
 282                 sfi['egress_ports'].append(sfi['egress'])
 283             del sfi['ingress']
 284             del sfi['egress']
 285             params = sfi.get('service_function_parameters')
 286             sfc_encap = False
 287             if params:
 288                 correlation = params.get('correlation')
 289                 if correlation:
 290                     sfc_encap = True
 291             sfi['sfc_encap'] = sfc_encap
 292             del sfi['service_function_parameters']
 293
 294     def __sf_os2mano(self, sf_list_dict):
 295         """Transform the openstack format (Port Pair Group) to mano format (SF)
 296         sf_list_dict can be a list of dict or a single dict
 297         """
 298         if isinstance(sf_list_dict, dict):
 299             sf_list_ = [sf_list_dict]
 300         elif isinstance(sf_list_dict, list):
 301             sf_list_ = sf_list_dict
 302         else:
 303             raise TypeError(
 304                 "param sf_list_dict must be a list or a dictionary")
 305         for sf in sf_list_:
 306             del sf['port_pair_group_parameters']
 307             sf['sfis'] = sf['port_pairs']
 308             del sf['port_pairs']
 309
 310     def __sfp_os2mano(self, sfp_list_dict):
 311         """Transform the openstack format (Port Chain) to mano format (SFP)
 312         sfp_list_dict can be a list of dict or a single dict
 313         """
 314         if isinstance(sfp_list_dict, dict):
 315             sfp_list_ = [sfp_list_dict]
 316         elif isinstance(sfp_list_dict, list):
 317             sfp_list_ = sfp_list_dict
 318         else:
 319             raise TypeError(
 320                 "param sfp_list_dict must be a list or a dictionary")
 321         for sfp in sfp_list_:
 322             params = sfp.pop('chain_parameters')
 323             sfc_encap = False
 324             if params:
 325                 correlation = params.get('correlation')
 326                 if correlation:
 327                     sfc_encap = True
 328             sfp['sfc_encap'] = sfc_encap
 329             sfp['spi'] = sfp.pop('chain_id')
 330             sfp['classifications'] = sfp.pop('flow_classifiers')
 331             sfp['service_functions'] = sfp.pop('port_pair_groups')
 332
 333     # placeholder for now; read TODO note below
 334     def _validate_classification(self, type, definition):
 335         # only legacy_flow_classifier Type is supported at this point
 336         return True
 337         # TODO(igordcard): this method should be an abstract method of an
 338         # abstract Classification class to be implemented by the specific
 339         # Types. Also, abstract vimconnector should call the validation
 340         # method before the implemented VIM connectors are called.
 341
 342     def _format_exception(self, exception):
 343         '''Transform a keystone, nova, neutron  exception into a vimconn exception'''
 344         if isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError,
 345                                   ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed
 346                                   )):
 347             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 348         elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException,
 349                                     neExceptions.NeutronException, nvExceptions.BadRequest)):
 350             raise vimconn.vimconnUnexpectedResponse(type(exception).__name__ + ": " + str(exception))
 351         elif isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound)):
 352             raise vimconn.vimconnNotFoundException(type(exception).__name__ + ": " + str(exception))
 353         elif isinstance(exception, nvExceptions.Conflict):
 354             raise vimconn.vimconnConflictException(type(exception).__name__ + ": " + str(exception))
 355         elif isinstance(exception, vimconn.vimconnException):
 356             raise exception
 357         else:  # ()
 358             self.logger.error("General Exception " + str(exception), exc_info=True)
 359             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 360
 361     def get_tenant_list(self, filter_dict={}):
 362         '''Obtain tenants of VIM
 363         filter_dict can contain the following keys:
 364             name: filter by tenant name
 365             id: filter by tenant uuid/id
 366             <other VIM specific>
 367         Returns the tenant list of dictionaries: [{'name':'<name>, 'id':'<id>, ...}, ...]
 368         '''
 369         self.logger.debug("Getting tenants from VIM filter: '%s'", str(filter_dict))
 370         try:
 371             self._reload_connection()
 372             if self.api_version3:
 373                 project_class_list = self.keystone.projects.list(name=filter_dict.get("name"))
 374             else:
 375                 project_class_list = self.keystone.tenants.findall(**filter_dict)
 376             project_list=[]
 377             for project in project_class_list:
 378                 if filter_dict.get('id') and filter_dict["id"] != project.id:
 379                     continue
 380                 project_list.append(project.to_dict())
 381             return project_list
 382         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError) as e:
 383             self._format_exception(e)
 384
 385     def new_tenant(self, tenant_name, tenant_description):
 386         '''Adds a new tenant to openstack VIM. Returns the tenant identifier'''
 387         self.logger.debug("Adding a new tenant name: %s", tenant_name)
 388         try:
 389             self._reload_connection()
 390             if self.api_version3:
 391                 project = self.keystone.projects.create(tenant_name, self.config.get("project_domain_id", "default"),
 392                                                         description=tenant_description, is_domain=False)
 393             else:
 394                 project = self.keystone.tenants.create(tenant_name, tenant_description)
 395             return project.id
 396         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 397             self._format_exception(e)
 398
 399     def delete_tenant(self, tenant_id):
 400         '''Delete a tenant from openstack VIM. Returns the old tenant identifier'''
 401         self.logger.debug("Deleting tenant %s from VIM", tenant_id)
 402         try:
 403             self._reload_connection()
 404             if self.api_version3:
 405                 self.keystone.projects.delete(tenant_id)
 406             else:
 407                 self.keystone.tenants.delete(tenant_id)
 408             return tenant_id
 409         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 410             self._format_exception(e)
 411
 412     def new_network(self,net_name, net_type, ip_profile=None, shared=False, vlan=None):
 413         '''Adds a tenant network to VIM. Returns the network identifier'''
 414         self.logger.debug("Adding a new network to VIM name '%s', type '%s'", net_name, net_type)
 415         #self.logger.debug(">>>>>>>>>>>>>>>>>> IP profile %s", str(ip_profile))
 416         try:
 417             new_net = None
 418             self._reload_connection()
 419             network_dict = {'name': net_name, 'admin_state_up': True}
 420             if net_type=="data" or net_type=="ptp":
 421                 if self.config.get('dataplane_physical_net') == None:
 422                     raise vimconn.vimconnConflictException("You must provide a 'dataplane_physical_net' at config value before creating sriov network")
 423                 network_dict["provider:physical_network"] = self.config['dataplane_physical_net'] #"physnet_sriov" #TODO physical
 424                 network_dict["provider:network_type"]     = "vlan"
 425                 if vlan!=None:
 426                     network_dict["provider:network_type"] = vlan
 427
 428                 ####### VIO Specific Changes #########
 429                 if self.vim_type == "VIO":
 430                     if vlan is not None:
 431                         network_dict["provider:segmentation_id"] = vlan
 432                     else:
 433                         if self.config.get('dataplane_net_vlan_range') is None:
 434                             raise vimconn.vimconnConflictException("You must provide "\
 435                                 "'dataplane_net_vlan_range' in format [start_ID - end_ID]"\
 436                                 "at config value before creating sriov network with vlan tag")
 437
 438                         network_dict["provider:segmentation_id"] = self._genrate_vlanID()
 439
 440             network_dict["shared"]=shared
 441             new_net=self.neutron.create_network({'network':network_dict})
 442             #print new_net
 443             #create subnetwork, even if there is no profile
 444             if not ip_profile:
 445                 ip_profile = {}
 446             if not ip_profile.get('subnet_address'):
 447                 #Fake subnet is required
 448                 subnet_rand = random.randint(0, 255)
 449                 ip_profile['subnet_address'] = "192.168.{}.0/24".format(subnet_rand)
 450             if 'ip_version' not in ip_profile:
 451                 ip_profile['ip_version'] = "IPv4"
 452             subnet = {"name":net_name+"-subnet",
 453                     "network_id": new_net["network"]["id"],
 454                     "ip_version": 4 if ip_profile['ip_version']=="IPv4" else 6,
 455                     "cidr": ip_profile['subnet_address']
 456                     }
 457             # Gateway should be set to None if not needed. Otherwise openstack assigns one by default
 458             if ip_profile.get('gateway_address'):
 459                 subnet['gateway_ip'] = ip_profile.get('gateway_address')
 460             if ip_profile.get('dns_address'):
 461                 subnet['dns_nameservers'] = ip_profile['dns_address'].split(";")
 462             if 'dhcp_enabled' in ip_profile:
 463                 subnet['enable_dhcp'] = False if \
 464                     ip_profile['dhcp_enabled']=="false" or ip_profile['dhcp_enabled']==False else True
 465             if ip_profile.get('dhcp_start_address'):
 466                 subnet['allocation_pools'] = []
 467                 subnet['allocation_pools'].append(dict())
 468                 subnet['allocation_pools'][0]['start'] = ip_profile['dhcp_start_address']
 469             if ip_profile.get('dhcp_count'):
 470                 #parts = ip_profile['dhcp_start_address'].split('.')
 471                 #ip_int = (int(parts[0]) << 24) + (int(parts[1]) << 16) + (int(parts[2]) << 8) + int(parts[3])
 472                 ip_int = int(netaddr.IPAddress(ip_profile['dhcp_start_address']))
 473                 ip_int += ip_profile['dhcp_count'] - 1
 474                 ip_str = str(netaddr.IPAddress(ip_int))
 475                 subnet['allocation_pools'][0]['end'] = ip_str
 476             #self.logger.debug(">>>>>>>>>>>>>>>>>> Subnet: %s", str(subnet))
 477             self.neutron.create_subnet({"subnet": subnet} )
 478             return new_net["network"]["id"]
 479         except Exception as e:
 480             if new_net:
 481                 self.neutron.delete_network(new_net['network']['id'])
 482             self._format_exception(e)
 483
 484     def get_network_list(self, filter_dict={}):
 485         '''Obtain tenant networks of VIM
 486         Filter_dict can be:
 487             name: network name
 488             id: network uuid
 489             shared: boolean
 490             tenant_id: tenant
 491             admin_state_up: boolean
 492             status: 'ACTIVE'
 493         Returns the network list of dictionaries
 494         '''
 495         self.logger.debug("Getting network from VIM filter: '%s'", str(filter_dict))
 496         try:
 497             self._reload_connection()
 498             filter_dict_os = filter_dict.copy()
 499             if self.api_version3 and "tenant_id" in filter_dict_os:
 500                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')  #T ODO check
 501             net_dict = self.neutron.list_networks(**filter_dict_os)
 502             net_list = net_dict["networks"]
 503             self.__net_os2mano(net_list)
 504             return net_list
 505         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 506             self._format_exception(e)
 507
 508     def get_network(self, net_id):
 509         '''Obtain details of network from VIM
 510         Returns the network information from a network id'''
 511         self.logger.debug(" Getting tenant network %s from VIM", net_id)
 512         filter_dict={"id": net_id}
 513         net_list = self.get_network_list(filter_dict)
 514         if len(net_list)==0:
 515             raise vimconn.vimconnNotFoundException("Network '{}' not found".format(net_id))
 516         elif len(net_list)>1:
 517             raise vimconn.vimconnConflictException("Found more than one network with this criteria")
 518         net = net_list[0]
 519         subnets=[]
 520         for subnet_id in net.get("subnets", () ):
 521             try:
 522                 subnet = self.neutron.show_subnet(subnet_id)
 523             except Exception as e:
 524                 self.logger.error("osconnector.get_network(): Error getting subnet %s %s" % (net_id, str(e)))
 525                 subnet = {"id": subnet_id, "fault": str(e)}
 526             subnets.append(subnet)
 527         net["subnets"] = subnets
 528         net["encapsulation"] = net.get('provider:network_type')
 529         net["segmentation_id"] = net.get('provider:segmentation_id')
 530         return net
 531
 532     def delete_network(self, net_id):
 533         '''Deletes a tenant network from VIM. Returns the old network identifier'''
 534         self.logger.debug("Deleting network '%s' from VIM", net_id)
 535         try:
 536             self._reload_connection()
 537             #delete VM ports attached to this networks before the network
 538             ports = self.neutron.list_ports(network_id=net_id)
 539             for p in ports['ports']:
 540                 try:
 541                     self.neutron.delete_port(p["id"])
 542                 except Exception as e:
 543                     self.logger.error("Error deleting port %s: %s", p["id"], str(e))
 544             self.neutron.delete_network(net_id)
 545             return net_id
 546         except (neExceptions.ConnectionFailed, neExceptions.NetworkNotFoundClient, neExceptions.NeutronException,
 547                 ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 548             self._format_exception(e)
 549
 550     def refresh_nets_status(self, net_list):
 551         '''Get the status of the networks
 552            Params: the list of network identifiers
 553            Returns a dictionary with:
 554                 net_id:         #VIM id of this network
 555                     status:     #Mandatory. Text with one of:
 556                                 #  DELETED (not found at vim)
 557                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 558                                 #  OTHER (Vim reported other status not understood)
 559                                 #  ERROR (VIM indicates an ERROR status)
 560                                 #  ACTIVE, INACTIVE, DOWN (admin down),
 561                                 #  BUILD (on building process)
 562                                 #
 563                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 564                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 565
 566         '''
 567         net_dict={}
 568         for net_id in net_list:
 569             net = {}
 570             try:
 571                 net_vim = self.get_network(net_id)
 572                 if net_vim['status'] in netStatus2manoFormat:
 573                     net["status"] = netStatus2manoFormat[ net_vim['status'] ]
 574                 else:
 575                     net["status"] = "OTHER"
 576                     net["error_msg"] = "VIM status reported " + net_vim['status']
 577
 578                 if net['status'] == "ACTIVE" and not net_vim['admin_state_up']:
 579                     net['status'] = 'DOWN'
 580                 try:
 581                     net['vim_info'] = yaml.safe_dump(net_vim, default_flow_style=True, width=256)
 582                 except yaml.representer.RepresenterError:
 583                     net['vim_info'] = str(net_vim)
 584                 if net_vim.get('fault'):  #TODO
 585                     net['error_msg'] = str(net_vim['fault'])
 586             except vimconn.vimconnNotFoundException as e:
 587                 self.logger.error("Exception getting net status: %s", str(e))
 588                 net['status'] = "DELETED"
 589                 net['error_msg'] = str(e)
 590             except vimconn.vimconnException as e:
 591                 self.logger.error("Exception getting net status: %s", str(e))
 592                 net['status'] = "VIM_ERROR"
 593                 net['error_msg'] = str(e)
 594             net_dict[net_id] = net
 595         return net_dict
 596
 597     def get_flavor(self, flavor_id):
 598         '''Obtain flavor details from the  VIM. Returns the flavor dict details'''
 599         self.logger.debug("Getting flavor '%s'", flavor_id)
 600         try:
 601             self._reload_connection()
 602             flavor = self.nova.flavors.find(id=flavor_id)
 603             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 604             return flavor.to_dict()
 605         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 606             self._format_exception(e)
 607
 608     def get_flavor_id_from_data(self, flavor_dict):
 609         """Obtain flavor id that match the flavor description
 610            Returns the flavor_id or raises a vimconnNotFoundException
 611            flavor_dict: contains the required ram, vcpus, disk
 612            If 'use_existing_flavors' is set to True at config, the closer flavor that provides same or more ram, vcpus
 613                 and disk is returned. Otherwise a flavor with exactly same ram, vcpus and disk is returned or a
 614                 vimconnNotFoundException is raised
 615         """
 616         exact_match = False if self.config.get('use_existing_flavors') else True
 617         try:
 618             self._reload_connection()
 619             flavor_candidate_id = None
 620             flavor_candidate_data = (10000, 10000, 10000)
 621             flavor_target = (flavor_dict["ram"], flavor_dict["vcpus"], flavor_dict["disk"])
 622             # numa=None
 623             numas = flavor_dict.get("extended", {}).get("numas")
 624             if numas:
 625                 #TODO
 626                 raise vimconn.vimconnNotFoundException("Flavor with EPA still not implemted")
 627                 # if len(numas) > 1:
 628                 #     raise vimconn.vimconnNotFoundException("Cannot find any flavor with more than one numa")
 629                 # numa=numas[0]
 630                 # numas = extended.get("numas")
 631             for flavor in self.nova.flavors.list():
 632                 epa = flavor.get_keys()
 633                 if epa:
 634                     continue
 635                     # TODO
 636                 flavor_data = (flavor.ram, flavor.vcpus, flavor.disk)
 637                 if flavor_data == flavor_target:
 638                     return flavor.id
 639                 elif not exact_match and flavor_target < flavor_data < flavor_candidate_data:
 640                     flavor_candidate_id = flavor.id
 641                     flavor_candidate_data = flavor_data
 642             if not exact_match and flavor_candidate_id:
 643                 return flavor_candidate_id
 644             raise vimconn.vimconnNotFoundException("Cannot find any flavor matching '{}'".format(str(flavor_dict)))
 645         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 646             self._format_exception(e)
 647
 648     def new_flavor(self, flavor_data, change_name_if_used=True):
 649         '''Adds a tenant flavor to openstack VIM
 650         if change_name_if_used is True, it will change name in case of conflict, because it is not supported name repetition
 651         Returns the flavor identifier
 652         '''
 653         self.logger.debug("Adding flavor '%s'", str(flavor_data))
 654         retry=0
 655         max_retries=3
 656         name_suffix = 0
 657         name=flavor_data['name']
 658         while retry<max_retries:
 659             retry+=1
 660             try:
 661                 self._reload_connection()
 662                 if change_name_if_used:
 663                     #get used names
 664                     fl_names=[]
 665                     fl=self.nova.flavors.list()
 666                     for f in fl:
 667                         fl_names.append(f.name)
 668                     while name in fl_names:
 669                         name_suffix += 1
 670                         name = flavor_data['name']+"-" + str(name_suffix)
 671
 672                 ram = flavor_data.get('ram',64)
 673                 vcpus = flavor_data.get('vcpus',1)
 674                 numa_properties=None
 675
 676                 extended = flavor_data.get("extended")
 677                 if extended:
 678                     numas=extended.get("numas")
 679                     if numas:
 680                         numa_nodes = len(numas)
 681                         if numa_nodes > 1:
 682                             return -1, "Can not add flavor with more than one numa"
 683                         numa_properties = {"hw:numa_nodes":str(numa_nodes)}
 684                         numa_properties["hw:mem_page_size"] = "large"
 685                         numa_properties["hw:cpu_policy"] = "dedicated"
 686                         numa_properties["hw:numa_mempolicy"] = "strict"
 687                         if self.vim_type == "VIO":
 688                             numa_properties["vmware:extra_config"] = '{"numa.nodeAffinity":"0"}'
 689                             numa_properties["vmware:latency_sensitivity_level"] = "high"
 690                         for numa in numas:
 691                             #overwrite ram and vcpus
 692                             #check if key 'memory' is present in numa else use ram value at flavor
 693                             if 'memory' in numa:
 694                                 ram = numa['memory']*1024
 695                             #See for reference: https://specs.openstack.org/openstack/nova-specs/specs/mitaka/implemented/virt-driver-cpu-thread-pinning.html
 696                             if 'paired-threads' in numa:
 697                                 vcpus = numa['paired-threads']*2
 698                                 #cpu_thread_policy "require" implies that the compute node must have an STM architecture
 699                                 numa_properties["hw:cpu_thread_policy"] = "require"
 700                                 numa_properties["hw:cpu_policy"] = "dedicated"
 701                             elif 'cores' in numa:
 702                                 vcpus = numa['cores']
 703                                 # cpu_thread_policy "prefer" implies that the host must not have an SMT architecture, or a non-SMT architecture will be emulated
 704                                 numa_properties["hw:cpu_thread_policy"] = "isolate"
 705                                 numa_properties["hw:cpu_policy"] = "dedicated"
 706                             elif 'threads' in numa:
 707                                 vcpus = numa['threads']
 708                                 # cpu_thread_policy "prefer" implies that the host may or may not have an SMT architecture
 709                                 numa_properties["hw:cpu_thread_policy"] = "prefer"
 710                                 numa_properties["hw:cpu_policy"] = "dedicated"
 711                             # for interface in numa.get("interfaces",() ):
 712                             #     if interface["dedicated"]=="yes":
 713                             #         raise vimconn.vimconnException("Passthrough interfaces are not supported for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable)
 714                             #     #TODO, add the key 'pci_passthrough:alias"="<label at config>:<number ifaces>"' when a way to connect it is available
 715
 716                 #create flavor
 717                 new_flavor=self.nova.flavors.create(name,
 718                                 ram,
 719                                 vcpus,
 720                                 flavor_data.get('disk',0),
 721                                 is_public=flavor_data.get('is_public', True)
 722                             )
 723                 #add metadata
 724                 if numa_properties:
 725                     new_flavor.set_keys(numa_properties)
 726                 return new_flavor.id
 727             except nvExceptions.Conflict as e:
 728                 if change_name_if_used and retry < max_retries:
 729                     continue
 730                 self._format_exception(e)
 731             #except nvExceptions.BadRequest as e:
 732             except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 733                 self._format_exception(e)
 734
 735     def delete_flavor(self,flavor_id):
 736         '''Deletes a tenant flavor from openstack VIM. Returns the old flavor_id
 737         '''
 738         try:
 739             self._reload_connection()
 740             self.nova.flavors.delete(flavor_id)
 741             return flavor_id
 742         #except nvExceptions.BadRequest as e:
 743         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 744             self._format_exception(e)
 745
 746     def new_image(self,image_dict):
 747         '''
 748         Adds a tenant image to VIM. imge_dict is a dictionary with:
 749             name: name
 750             disk_format: qcow2, vhd, vmdk, raw (by default), ...
 751             location: path or URI
 752             public: "yes" or "no"
 753             metadata: metadata of the image
 754         Returns the image_id
 755         '''
 756         retry=0
 757         max_retries=3
 758         while retry<max_retries:
 759             retry+=1
 760             try:
 761                 self._reload_connection()
 762                 #determine format  http://docs.openstack.org/developer/glance/formats.html
 763                 if "disk_format" in image_dict:
 764                     disk_format=image_dict["disk_format"]
 765                 else: #autodiscover based on extension
 766                     if image_dict['location'][-6:]==".qcow2":
 767                         disk_format="qcow2"
 768                     elif image_dict['location'][-4:]==".vhd":
 769                         disk_format="vhd"
 770                     elif image_dict['location'][-5:]==".vmdk":
 771                         disk_format="vmdk"
 772                     elif image_dict['location'][-4:]==".vdi":
 773                         disk_format="vdi"
 774                     elif image_dict['location'][-4:]==".iso":
 775                         disk_format="iso"
 776                     elif image_dict['location'][-4:]==".aki":
 777                         disk_format="aki"
 778                     elif image_dict['location'][-4:]==".ari":
 779                         disk_format="ari"
 780                     elif image_dict['location'][-4:]==".ami":
 781                         disk_format="ami"
 782                     else:
 783                         disk_format="raw"
 784                 self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location'])
 785                 if image_dict['location'][0:4]=="http":
 786                     new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 787                             container_format="bare", location=image_dict['location'], disk_format=disk_format)
 788                 else: #local path
 789                     with open(image_dict['location']) as fimage:
 790                         new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 791                             container_format="bare", data=fimage, disk_format=disk_format)
 792                 #insert metadata. We cannot use 'new_image.properties.setdefault'
 793                 #because nova and glance are "INDEPENDENT" and we are using nova for reading metadata
 794                 new_image_nova=self.nova.images.find(id=new_image.id)
 795                 new_image_nova.metadata.setdefault('location',image_dict['location'])
 796                 metadata_to_load = image_dict.get('metadata')
 797                 if metadata_to_load:
 798                     for k,v in yaml.load(metadata_to_load).iteritems():
 799                         new_image_nova.metadata.setdefault(k,v)
 800                 return new_image.id
 801             except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e:
 802                 self._format_exception(e)
 803             except (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 804                 if retry==max_retries:
 805                     continue
 806                 self._format_exception(e)
 807             except IOError as e:  #can not open the file
 808                 raise vimconn.vimconnConnectionException(type(e).__name__ + ": " + str(e)+ " for " + image_dict['location'],
 809                                                          http_code=vimconn.HTTP_Bad_Request)
 810
 811     def delete_image(self, image_id):
 812         '''Deletes a tenant image from openstack VIM. Returns the old id
 813         '''
 814         try:
 815             self._reload_connection()
 816             self.nova.images.delete(image_id)
 817             return image_id
 818         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e: #TODO remove
 819             self._format_exception(e)
 820
 821     def get_image_id_from_path(self, path):
 822         '''Get the image id from image path in the VIM database. Returns the image_id'''
 823         try:
 824             self._reload_connection()
 825             images = self.nova.images.list()
 826             for image in images:
 827                 if image.metadata.get("location")==path:
 828                     return image.id
 829             raise vimconn.vimconnNotFoundException("image with location '{}' not found".format( path))
 830         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 831             self._format_exception(e)
 832
 833     def get_image_list(self, filter_dict={}):
 834         '''Obtain tenant images from VIM
 835         Filter_dict can be:
 836             id: image id
 837             name: image name
 838             checksum: image checksum
 839         Returns the image list of dictionaries:
 840             [{<the fields at Filter_dict plus some VIM specific>}, ...]
 841             List can be empty
 842         '''
 843         self.logger.debug("Getting image list from VIM filter: '%s'", str(filter_dict))
 844         try:
 845             self._reload_connection()
 846             filter_dict_os = filter_dict.copy()
 847             #First we filter by the available filter fields: name, id. The others are removed.
 848             filter_dict_os.pop('checksum', None)
 849             image_list = self.nova.images.findall(**filter_dict_os)
 850             if len(image_list) == 0:
 851                 return []
 852             #Then we filter by the rest of filter fields: checksum
 853             filtered_list = []
 854             for image in image_list:
 855                 try:
 856                     image_class = self.glance.images.get(image.id)
 857                     if 'checksum' not in filter_dict or image_class['checksum'] == filter_dict.get('checksum'):
 858                         filtered_list.append(image_class.copy())
 859                 except gl1Exceptions.HTTPNotFound:
 860                     pass
 861             return filtered_list
 862         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 863             self._format_exception(e)
 864
 865     def __wait_for_vm(self, vm_id, status):
 866         """wait until vm is in the desired status and return True.
 867         If the VM gets in ERROR status, return false.
 868         If the timeout is reached generate an exception"""
 869         elapsed_time = 0
 870         while elapsed_time < server_timeout:
 871             vm_status = self.nova.servers.get(vm_id).status
 872             if vm_status == status:
 873                 return True
 874             if vm_status == 'ERROR':
 875                 return False
 876             time.sleep(1)
 877             elapsed_time += 1
 878
 879         # if we exceeded the timeout rollback
 880         if elapsed_time >= server_timeout:
 881             raise vimconn.vimconnException('Timeout waiting for instance ' + vm_id + ' to get ' + status,
 882                                            http_code=vimconn.HTTP_Request_Timeout)
 883
 884     def _get_openstack_availablity_zones(self):
 885         """
 886         Get from openstack availability zones available
 887         :return:
 888         """
 889         try:
 890             openstack_availability_zone = self.nova.availability_zones.list()
 891             openstack_availability_zone = [str(zone.zoneName) for zone in openstack_availability_zone
 892                                            if zone.zoneName != 'internal']
 893             return openstack_availability_zone
 894         except Exception as e:
 895             return None
 896
 897     def _set_availablity_zones(self):
 898         """
 899         Set vim availablity zone
 900         :return:
 901         """
 902
 903         if 'availability_zone' in self.config:
 904             vim_availability_zones = self.config.get('availability_zone')
 905             if isinstance(vim_availability_zones, str):
 906                 self.availability_zone = [vim_availability_zones]
 907             elif isinstance(vim_availability_zones, list):
 908                 self.availability_zone = vim_availability_zones
 909         else:
 910             self.availability_zone = self._get_openstack_availablity_zones()
 911
 912     def _get_vm_availability_zone(self, availability_zone_index, availability_zone_list):
 913         """
 914         Return thge availability zone to be used by the created VM.
 915         :return: The VIM availability zone to be used or None
 916         """
 917         if availability_zone_index is None:
 918             if not self.config.get('availability_zone'):
 919                 return None
 920             elif isinstance(self.config.get('availability_zone'), str):
 921                 return self.config['availability_zone']
 922             else:
 923                 # TODO consider using a different parameter at config for default AV and AV list match
 924                 return self.config['availability_zone'][0]
 925
 926         vim_availability_zones = self.availability_zone
 927         # check if VIM offer enough availability zones describe in the VNFD
 928         if vim_availability_zones and len(availability_zone_list) <= len(vim_availability_zones):
 929             # check if all the names of NFV AV match VIM AV names
 930             match_by_index = False
 931             for av in availability_zone_list:
 932                 if av not in vim_availability_zones:
 933                     match_by_index = True
 934                     break
 935             if match_by_index:
 936                 return vim_availability_zones[availability_zone_index]
 937             else:
 938                 return availability_zone_list[availability_zone_index]
 939         else:
 940             raise vimconn.vimconnConflictException("No enough availability zones at VIM for this deployment")
 941
 942     def new_vminstance(self, name, description, start, image_id, flavor_id, net_list, cloud_config=None, disk_list=None,
 943                        availability_zone_index=None, availability_zone_list=None):
 944         """Adds a VM instance to VIM
 945         Params:
 946             start: indicates if VM must start or boot in pause mode. Ignored
 947             image_id,flavor_id: iamge and flavor uuid
 948             net_list: list of interfaces, each one is a dictionary with:
 949                 name:
 950                 net_id: network uuid to connect
 951                 vpci: virtual vcpi to assign, ignored because openstack lack #TODO
 952                 model: interface model, ignored #TODO
 953                 mac_address: used for  SR-IOV ifaces #TODO for other types
 954                 use: 'data', 'bridge',  'mgmt'
 955                 type: 'virtual', 'PCI-PASSTHROUGH'('PF'), 'SR-IOV'('VF'), 'VFnotShared'
 956                 vim_id: filled/added by this function
 957                 floating_ip: True/False (or it can be None)
 958             'cloud_config': (optional) dictionary with:
 959             'key-pairs': (optional) list of strings with the public key to be inserted to the default user
 960             'users': (optional) list of users to be inserted, each item is a dict with:
 961                 'name': (mandatory) user name,
 962                 'key-pairs': (optional) list of strings with the public key to be inserted to the user
 963             'user-data': (optional) string is a text script to be passed directly to cloud-init
 964             'config-files': (optional). List of files to be transferred. Each item is a dict with:
 965                 'dest': (mandatory) string with the destination absolute path
 966                 'encoding': (optional, by default text). Can be one of:
 967                     'b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64'
 968                 'content' (mandatory): string with the content of the file
 969                 'permissions': (optional) string with file permissions, typically octal notation '0644'
 970                 'owner': (optional) file owner, string with the format 'owner:group'
 971             'boot-data-drive': boolean to indicate if user-data must be passed using a boot drive (hard disk)
 972             'disk_list': (optional) list with additional disks to the VM. Each item is a dict with:
 973                 'image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted
 974                 'size': (mandatory) string with the size of the disk in GB
 975             availability_zone_index: Index of availability_zone_list to use for this this VM. None if not AV required
 976             availability_zone_list: list of availability zones given by user in the VNFD descriptor.  Ignore if
 977                 availability_zone_index is None
 978                 #TODO ip, security groups
 979         Returns a tuple with the instance identifier and created_items or raises an exception on error
 980             created_items can be None or a dictionary where this method can include key-values that will be passed to
 981             the method delete_vminstance and action_vminstance. Can be used to store created ports, volumes, etc.
 982             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
 983             as not present.
 984         """
 985         self.logger.debug("new_vminstance input: image='%s' flavor='%s' nics='%s'",image_id, flavor_id,str(net_list))
 986         try:
 987             server = None
 988             created_items = {}
 989             # metadata = {}
 990             net_list_vim = []
 991             external_network = []   # list of external networks to be connected to instance, later on used to create floating_ip
 992             no_secured_ports = []   # List of port-is with port-security disabled
 993             self._reload_connection()
 994             # metadata_vpci = {}   # For a specific neutron plugin
 995             block_device_mapping = None
 996             for net in net_list:
 997                 if not net.get("net_id"):   # skip non connected iface
 998                     continue
 999
1000                 port_dict={
1001                     "network_id": net["net_id"],
1002                     "name": net.get("name"),
1003                     "admin_state_up": True
1004                 }
1005                 if net["type"]=="virtual":
1006                     pass
1007                     # if "vpci" in net:
1008                     #     metadata_vpci[ net["net_id"] ] = [[ net["vpci"], "" ]]
1009                 elif net["type"] == "VF" or net["type"] == "SR-IOV":  # for VF
1010                     # if "vpci" in net:
1011                     #     if "VF" not in metadata_vpci:
1012                     #         metadata_vpci["VF"]=[]
1013                     #     metadata_vpci["VF"].append([ net["vpci"], "" ])
1014                     port_dict["binding:vnic_type"]="direct"
1015                     # VIO specific Changes
1016                     if self.vim_type == "VIO":
1017                         # Need to create port with port_security_enabled = False and no-security-groups
1018                         port_dict["port_security_enabled"]=False
1019                         port_dict["provider_security_groups"]=[]
1020                         port_dict["security_groups"]=[]
1021                 else:   # For PT PCI-PASSTHROUGH
1022                     # VIO specific Changes
1023                     # Current VIO release does not support port with type 'direct-physical'
1024                     # So no need to create virtual port in case of PCI-device.
1025                     # Will update port_dict code when support gets added in next VIO release
1026                     if self.vim_type == "VIO":
1027                         raise vimconn.vimconnNotSupportedException(
1028                             "Current VIO release does not support full passthrough (PT)")
1029                     # if "vpci" in net:
1030                     #     if "PF" not in metadata_vpci:
1031                     #         metadata_vpci["PF"]=[]
1032                     #     metadata_vpci["PF"].append([ net["vpci"], "" ])
1033                     port_dict["binding:vnic_type"]="direct-physical"
1034                 if not port_dict["name"]:
1035                     port_dict["name"]=name
1036                 if net.get("mac_address"):
1037                     port_dict["mac_address"]=net["mac_address"]
1038                 if net.get("ip_address"):
1039                     port_dict["fixed_ips"] = [{'ip_address': net["ip_address"]}]
1040                     # TODO add 'subnet_id': <subnet_id>
1041                 new_port = self.neutron.create_port({"port": port_dict })
1042                 created_items["port:" + str(new_port["port"]["id"])] = True
1043                 net["mac_adress"] = new_port["port"]["mac_address"]
1044                 net["vim_id"] = new_port["port"]["id"]
1045                 # if try to use a network without subnetwork, it will return a emtpy list
1046                 fixed_ips = new_port["port"].get("fixed_ips")
1047                 if fixed_ips:
1048                     net["ip"] = fixed_ips[0].get("ip_address")
1049                 else:
1050                     net["ip"] = None
1051
1052                 port = {"port-id": new_port["port"]["id"]}
1053                 if float(self.nova.api_version.get_string()) >= 2.32:
1054                     port["tag"] = new_port["port"]["name"]
1055                 net_list_vim.append(port)
1056
1057                 if net.get('floating_ip', False):
1058                     net['exit_on_floating_ip_error'] = True
1059                     external_network.append(net)
1060                 elif net['use'] == 'mgmt' and self.config.get('use_floating_ip'):
1061                     net['exit_on_floating_ip_error'] = False
1062                     external_network.append(net)
1063                     net['floating_ip'] = self.config.get('use_floating_ip')
1064
1065                 # If port security is disabled when the port has not yet been attached to the VM, then all vm traffic is dropped.
1066                 # As a workaround we wait until the VM is active and then disable the port-security
1067                 if net.get("port_security") == False:
1068                     no_secured_ports.append(new_port["port"]["id"])
1069
1070             # if metadata_vpci:
1071             #     metadata = {"pci_assignement": json.dumps(metadata_vpci)}
1072             #     if len(metadata["pci_assignement"]) >255:
1073             #         #limit the metadata size
1074             #         #metadata["pci_assignement"] = metadata["pci_assignement"][0:255]
1075             #         self.logger.warn("Metadata deleted since it exceeds the expected length (255) ")
1076             #         metadata = {}
1077
1078             self.logger.debug("name '%s' image_id '%s'flavor_id '%s' net_list_vim '%s' description '%s'",
1079                               name, image_id, flavor_id, str(net_list_vim), description)
1080
1081             security_groups = self.config.get('security_groups')
1082             if type(security_groups) is str:
1083                 security_groups = ( security_groups, )
1084             # cloud config
1085             config_drive, userdata = self._create_user_data(cloud_config)
1086
1087             # Create additional volumes in case these are present in disk_list
1088             base_disk_index = ord('b')
1089             if disk_list != None:
1090                 block_device_mapping = {}
1091                 for disk in disk_list:
1092                     if 'image_id' in disk:
1093                         volume = self.cinder.volumes.create(size = disk['size'],name = name + '_vd' +
1094                                     chr(base_disk_index), imageRef = disk['image_id'])
1095                     else:
1096                         volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1097                                     chr(base_disk_index))
1098                     created_items["volume:" + str(volume.id)] = True
1099                     block_device_mapping['_vd' +  chr(base_disk_index)] = volume.id
1100                     base_disk_index += 1
1101
1102                 # Wait until volumes are with status available
1103                 keep_waiting = True
1104                 elapsed_time = 0
1105                 while keep_waiting and elapsed_time < volume_timeout:
1106                     keep_waiting = False
1107                     for volume_id in block_device_mapping.itervalues():
1108                         if self.cinder.volumes.get(volume_id).status != 'available':
1109                             keep_waiting = True
1110                     if keep_waiting:
1111                         time.sleep(1)
1112                         elapsed_time += 1
1113
1114                 # If we exceeded the timeout rollback
1115                 if elapsed_time >= volume_timeout:
1116                     raise vimconn.vimconnException('Timeout creating volumes for instance ' + name,
1117                                                    http_code=vimconn.HTTP_Request_Timeout)
1118             # get availability Zone
1119             vm_av_zone = self._get_vm_availability_zone(availability_zone_index, availability_zone_list)
1120
1121             self.logger.debug("nova.servers.create({}, {}, {}, nics={}, security_groups={}, "
1122                               "availability_zone={}, key_name={}, userdata={}, config_drive={}, "
1123                               "block_device_mapping={})".format(name, image_id, flavor_id, net_list_vim,
1124                                                                 security_groups, vm_av_zone, self.config.get('keypair'),
1125                                                                 userdata, config_drive, block_device_mapping))
1126             server = self.nova.servers.create(name, image_id, flavor_id, nics=net_list_vim,
1127                                               security_groups=security_groups,
1128                                               availability_zone=vm_av_zone,
1129                                               key_name=self.config.get('keypair'),
1130                                               userdata=userdata,
1131                                               config_drive=config_drive,
1132                                               block_device_mapping=block_device_mapping
1133                                               )  # , description=description)
1134
1135             vm_start_time = time.time()
1136             # Previously mentioned workaround to wait until the VM is active and then disable the port-security
1137             if no_secured_ports:
1138                 self.__wait_for_vm(server.id, 'ACTIVE')
1139
1140             for port_id in no_secured_ports:
1141                 try:
1142                     self.neutron.update_port(port_id, {"port": {"port_security_enabled": False, "security_groups": None} })
1143                 except Exception as e:
1144                     self.logger.error("It was not possible to disable port security for port {}".format(port_id))
1145                     raise
1146
1147             # print "DONE :-)", server
1148
1149             pool_id = None
1150             if external_network:
1151                 floating_ips = self.neutron.list_floatingips().get("floatingips", ())
1152             for floating_network in external_network:
1153                 try:
1154                     assigned = False
1155                     while not assigned:
1156                         if floating_ips:
1157                             ip = floating_ips.pop(0)
1158                             if ip.get("port_id", False) or ip.get('tenant_id') != server.tenant_id:
1159                                 continue
1160                             if isinstance(floating_network['floating_ip'], str):
1161                                 if ip.get("floating_network_id") != floating_network['floating_ip']:
1162                                     continue
1163                             free_floating_ip = ip.get("floating_ip_address")
1164                         else:
1165                             if isinstance(floating_network['floating_ip'], str):
1166                                 pool_id = floating_network['floating_ip']
1167                             else:
1168                                 #Find the external network
1169                                 external_nets = list()
1170                                 for net in self.neutron.list_networks()['networks']:
1171                                     if net['router:external']:
1172                                             external_nets.append(net)
1173
1174                                 if len(external_nets) == 0:
1175                                     raise vimconn.vimconnException("Cannot create floating_ip automatically since no external "
1176                                                                    "network is present",
1177                                                                     http_code=vimconn.HTTP_Conflict)
1178                                 if len(external_nets) > 1:
1179                                     raise vimconn.vimconnException("Cannot create floating_ip automatically since multiple "
1180                                                                    "external networks are present",
1181                                                                    http_code=vimconn.HTTP_Conflict)
1182
1183                                 pool_id = external_nets[0].get('id')
1184                             param = {'floatingip': {'floating_network_id': pool_id, 'tenant_id': server.tenant_id}}
1185                             try:
1186                                 #self.logger.debug("Creating floating IP")
1187                                 new_floating_ip = self.neutron.create_floatingip(param)
1188                                 free_floating_ip = new_floating_ip['floatingip']['floating_ip_address']
1189                             except Exception as e:
1190                                 raise vimconn.vimconnException(type(e).__name__ + ": Cannot create new floating_ip " +
1191                                                                str(e), http_code=vimconn.HTTP_Conflict)
1192
1193                         fix_ip = floating_network.get('ip')
1194                         while not assigned:
1195                             try:
1196                                 server.add_floating_ip(free_floating_ip, fix_ip)
1197                                 assigned = True
1198                             except Exception as e:
1199                                 vm_status = self.nova.servers.get(server.id).status
1200                                 if vm_status != 'ACTIVE' and vm_status != 'ERROR':
1201                                     if time.time() - vm_start_time < server_timeout:
1202                                         time.sleep(5)
1203                                         continue
1204                                 raise vimconn.vimconnException(type(e).__name__ + ": Cannot create floating_ip "+  str(e),
1205                                                                http_code=vimconn.HTTP_Conflict)
1206
1207                 except Exception as e:
1208                     if not floating_network['exit_on_floating_ip_error']:
1209                         self.logger.warn("Cannot create floating_ip. %s", str(e))
1210                         continue
1211                     raise
1212
1213             return server.id, created_items
1214 #        except nvExceptions.NotFound as e:
1215 #            error_value=-vimconn.HTTP_Not_Found
1216 #            error_text= "vm instance %s not found" % vm_id
1217 #        except TypeError as e:
1218 #            raise vimconn.vimconnException(type(e).__name__ + ": "+  str(e), http_code=vimconn.HTTP_Bad_Request)
1219
1220         except Exception as e:
1221             server_id = None
1222             if server:
1223                 server_id = server.id
1224             try:
1225                 self.delete_vminstance(server_id, created_items)
1226             except Exception as e2:
1227                 self.logger.error("new_vminstance rollback fail {}".format(e2))
1228
1229             self._format_exception(e)
1230
1231     def get_vminstance(self,vm_id):
1232         '''Returns the VM instance information from VIM'''
1233         #self.logger.debug("Getting VM from VIM")
1234         try:
1235             self._reload_connection()
1236             server = self.nova.servers.find(id=vm_id)
1237             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
1238             return server.to_dict()
1239         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1240             self._format_exception(e)
1241
1242     def get_vminstance_console(self,vm_id, console_type="vnc"):
1243         '''
1244         Get a console for the virtual machine
1245         Params:
1246             vm_id: uuid of the VM
1247             console_type, can be:
1248                 "novnc" (by default), "xvpvnc" for VNC types,
1249                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
1250         Returns dict with the console parameters:
1251                 protocol: ssh, ftp, http, https, ...
1252                 server:   usually ip address
1253                 port:     the http, ssh, ... port
1254                 suffix:   extra text, e.g. the http path and query string
1255         '''
1256         self.logger.debug("Getting VM CONSOLE from VIM")
1257         try:
1258             self._reload_connection()
1259             server = self.nova.servers.find(id=vm_id)
1260             if console_type == None or console_type == "novnc":
1261                 console_dict = server.get_vnc_console("novnc")
1262             elif console_type == "xvpvnc":
1263                 console_dict = server.get_vnc_console(console_type)
1264             elif console_type == "rdp-html5":
1265                 console_dict = server.get_rdp_console(console_type)
1266             elif console_type == "spice-html5":
1267                 console_dict = server.get_spice_console(console_type)
1268             else:
1269                 raise vimconn.vimconnException("console type '{}' not allowed".format(console_type), http_code=vimconn.HTTP_Bad_Request)
1270
1271             console_dict1 = console_dict.get("console")
1272             if console_dict1:
1273                 console_url = console_dict1.get("url")
1274                 if console_url:
1275                     #parse console_url
1276                     protocol_index = console_url.find("//")
1277                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1278                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1279                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1280                         return -vimconn.HTTP_Internal_Server_Error, "Unexpected response from VIM"
1281                     console_dict={"protocol": console_url[0:protocol_index],
1282                                   "server":   console_url[protocol_index+2:port_index],
1283                                   "port":     console_url[port_index:suffix_index],
1284                                   "suffix":   console_url[suffix_index+1:]
1285                                   }
1286                     protocol_index += 2
1287                     return console_dict
1288             raise vimconn.vimconnUnexpectedResponse("Unexpected response from VIM")
1289
1290         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.BadRequest, ConnectionError) as e:
1291             self._format_exception(e)
1292
1293     def delete_vminstance(self, vm_id, created_items=None):
1294         '''Removes a VM instance from VIM. Returns the old identifier
1295         '''
1296         #print "osconnector: Getting VM from VIM"
1297         if created_items == None:
1298             created_items = {}
1299         try:
1300             self._reload_connection()
1301             # delete VM ports attached to this networks before the virtual machine
1302             for k, v in created_items.items():
1303                 if not v:  # skip already deleted
1304                     continue
1305                 try:
1306                     k_item, _, k_id = k.partition(":")
1307                     if k_item == "port":
1308                         self.neutron.delete_port(k_id)
1309                 except Exception as e:
1310                     self.logger.error("Error deleting port: {}: {}".format(type(e).__name__, e))
1311
1312             # #commented because detaching the volumes makes the servers.delete not work properly ?!?
1313             # #dettach volumes attached
1314             # server = self.nova.servers.get(vm_id)
1315             # volumes_attached_dict = server._info['os-extended-volumes:volumes_attached']   #volume['id']
1316             # #for volume in volumes_attached_dict:
1317             # #    self.cinder.volumes.detach(volume['id'])
1318
1319             if vm_id:
1320                 self.nova.servers.delete(vm_id)
1321
1322             # delete volumes. Although having detached, they should have in active status before deleting
1323             # we ensure in this loop
1324             keep_waiting = True
1325             elapsed_time = 0
1326             while keep_waiting and elapsed_time < volume_timeout:
1327                 keep_waiting = False
1328                 for k, v in created_items.items():
1329                     if not v:  # skip already deleted
1330                         continue
1331                     try:
1332                         k_item, _, k_id = k.partition(":")
1333                         if k_item == "volume":
1334                             if self.cinder.volumes.get(k_id).status != 'available':
1335                                 keep_waiting = True
1336                             else:
1337                                 self.cinder.volumes.delete(k_id)
1338                     except Exception as e:
1339                         self.logger.error("Error deleting volume: {}: {}".format(type(e).__name__, e))
1340                 if keep_waiting:
1341                     time.sleep(1)
1342                     elapsed_time += 1
1343             return None
1344         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
1345             self._format_exception(e)
1346
1347     def refresh_vms_status(self, vm_list):
1348         '''Get the status of the virtual machines and their interfaces/ports
1349            Params: the list of VM identifiers
1350            Returns a dictionary with:
1351                 vm_id:          #VIM id of this Virtual Machine
1352                     status:     #Mandatory. Text with one of:
1353                                 #  DELETED (not found at vim)
1354                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1355                                 #  OTHER (Vim reported other status not understood)
1356                                 #  ERROR (VIM indicates an ERROR status)
1357                                 #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
1358                                 #  CREATING (on building process), ERROR
1359                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
1360                                 #
1361                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
1362                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
1363                     interfaces:
1364                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
1365                         mac_address:      #Text format XX:XX:XX:XX:XX:XX
1366                         vim_net_id:       #network id where this interface is connected
1367                         vim_interface_id: #interface/port VIM id
1368                         ip_address:       #null, or text with IPv4, IPv6 address
1369                         compute_node:     #identification of compute node where PF,VF interface is allocated
1370                         pci:              #PCI address of the NIC that hosts the PF,VF
1371                         vlan:             #physical VLAN used for VF
1372         '''
1373         vm_dict={}
1374         self.logger.debug("refresh_vms status: Getting tenant VM instance information from VIM")
1375         for vm_id in vm_list:
1376             vm={}
1377             try:
1378                 vm_vim = self.get_vminstance(vm_id)
1379                 if vm_vim['status'] in vmStatus2manoFormat:
1380                     vm['status']    =  vmStatus2manoFormat[ vm_vim['status'] ]
1381                 else:
1382                     vm['status']    = "OTHER"
1383                     vm['error_msg'] = "VIM status reported " + vm_vim['status']
1384                 try:
1385                     vm['vim_info']  = yaml.safe_dump(vm_vim, default_flow_style=True, width=256)
1386                 except yaml.representer.RepresenterError:
1387                     vm['vim_info'] = str(vm_vim)
1388                 vm["interfaces"] = []
1389                 if vm_vim.get('fault'):
1390                     vm['error_msg'] = str(vm_vim['fault'])
1391                 #get interfaces
1392                 try:
1393                     self._reload_connection()
1394                     port_dict=self.neutron.list_ports(device_id=vm_id)
1395                     for port in port_dict["ports"]:
1396                         interface={}
1397                         try:
1398                             interface['vim_info'] = yaml.safe_dump(port, default_flow_style=True, width=256)
1399                         except yaml.representer.RepresenterError:
1400                             interface['vim_info'] = str(port)
1401                         interface["mac_address"] = port.get("mac_address")
1402                         interface["vim_net_id"] = port["network_id"]
1403                         interface["vim_interface_id"] = port["id"]
1404                         # check if OS-EXT-SRV-ATTR:host is there,
1405                         # in case of non-admin credentials, it will be missing
1406                         if vm_vim.get('OS-EXT-SRV-ATTR:host'):
1407                             interface["compute_node"] = vm_vim['OS-EXT-SRV-ATTR:host']
1408                         interface["pci"] = None
1409
1410                         # check if binding:profile is there,
1411                         # in case of non-admin credentials, it will be missing
1412                         if port.get('binding:profile'):
1413                             if port['binding:profile'].get('pci_slot'):
1414                                 # TODO: At the moment sr-iov pci addresses are converted to PF pci addresses by setting the slot to 0x00
1415                                 # TODO: This is just a workaround valid for niantinc. Find a better way to do so
1416                                 #   CHANGE DDDD:BB:SS.F to DDDD:BB:00.(F%2)   assuming there are 2 ports per nic
1417                                 pci = port['binding:profile']['pci_slot']
1418                                 # interface["pci"] = pci[:-4] + "00." + str(int(pci[-1]) % 2)
1419                                 interface["pci"] = pci
1420                         interface["vlan"] = None
1421                         #if network is of type vlan and port is of type direct (sr-iov) then set vlan id
1422                         network = self.neutron.show_network(port["network_id"])
1423                         if network['network'].get('provider:network_type') == 'vlan' and \
1424                             port.get("binding:vnic_type") == "direct":
1425                             interface["vlan"] = network['network'].get('provider:segmentation_id')
1426                         ips=[]
1427                         #look for floating ip address
1428                         floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
1429                         if floating_ip_dict.get("floatingips"):
1430                             ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address") )
1431
1432                         for subnet in port["fixed_ips"]:
1433                             ips.append(subnet["ip_address"])
1434                         interface["ip_address"] = ";".join(ips)
1435                         vm["interfaces"].append(interface)
1436                 except Exception as e:
1437                     self.logger.error("Error getting vm interface information " + type(e).__name__ + ": "+  str(e))
1438             except vimconn.vimconnNotFoundException as e:
1439                 self.logger.error("Exception getting vm status: %s", str(e))
1440                 vm['status'] = "DELETED"
1441                 vm['error_msg'] = str(e)
1442             except vimconn.vimconnException as e:
1443                 self.logger.error("Exception getting vm status: %s", str(e))
1444                 vm['status'] = "VIM_ERROR"
1445                 vm['error_msg'] = str(e)
1446             vm_dict[vm_id] = vm
1447         return vm_dict
1448
1449     def action_vminstance(self, vm_id, action_dict, created_items={}):
1450         '''Send and action over a VM instance from VIM
1451         Returns None or the console dict if the action was successfully sent to the VIM'''
1452         self.logger.debug("Action over VM '%s': %s", vm_id, str(action_dict))
1453         try:
1454             self._reload_connection()
1455             server = self.nova.servers.find(id=vm_id)
1456             if "start" in action_dict:
1457                 if action_dict["start"]=="rebuild":
1458                     server.rebuild()
1459                 else:
1460                     if server.status=="PAUSED":
1461                         server.unpause()
1462                     elif server.status=="SUSPENDED":
1463                         server.resume()
1464                     elif server.status=="SHUTOFF":
1465                         server.start()
1466             elif "pause" in action_dict:
1467                 server.pause()
1468             elif "resume" in action_dict:
1469                 server.resume()
1470             elif "shutoff" in action_dict or "shutdown" in action_dict:
1471                 server.stop()
1472             elif "forceOff" in action_dict:
1473                 server.stop() #TODO
1474             elif "terminate" in action_dict:
1475                 server.delete()
1476             elif "createImage" in action_dict:
1477                 server.create_image()
1478                 #"path":path_schema,
1479                 #"description":description_schema,
1480                 #"name":name_schema,
1481                 #"metadata":metadata_schema,
1482                 #"imageRef": id_schema,
1483                 #"disk": {"oneOf":[{"type": "null"}, {"type":"string"}] },
1484             elif "rebuild" in action_dict:
1485                 server.rebuild(server.image['id'])
1486             elif "reboot" in action_dict:
1487                 server.reboot() #reboot_type='SOFT'
1488             elif "console" in action_dict:
1489                 console_type = action_dict["console"]
1490                 if console_type == None or console_type == "novnc":
1491                     console_dict = server.get_vnc_console("novnc")
1492                 elif console_type == "xvpvnc":
1493                     console_dict = server.get_vnc_console(console_type)
1494                 elif console_type == "rdp-html5":
1495                     console_dict = server.get_rdp_console(console_type)
1496                 elif console_type == "spice-html5":
1497                     console_dict = server.get_spice_console(console_type)
1498                 else:
1499                     raise vimconn.vimconnException("console type '{}' not allowed".format(console_type),
1500                                                    http_code=vimconn.HTTP_Bad_Request)
1501                 try:
1502                     console_url = console_dict["console"]["url"]
1503                     #parse console_url
1504                     protocol_index = console_url.find("//")
1505                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1506                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1507                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1508                         raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1509                     console_dict2={"protocol": console_url[0:protocol_index],
1510                                   "server":   console_url[protocol_index+2 : port_index],
1511                                   "port":     int(console_url[port_index+1 : suffix_index]),
1512                                   "suffix":   console_url[suffix_index+1:]
1513                                   }
1514                     return console_dict2
1515                 except Exception as e:
1516                     raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1517
1518             return None
1519         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1520             self._format_exception(e)
1521         #TODO insert exception vimconn.HTTP_Unauthorized
1522
1523     ####### VIO Specific Changes #########
1524     def _genrate_vlanID(self):
1525         """
1526          Method to get unused vlanID
1527             Args:
1528                 None
1529             Returns:
1530                 vlanID
1531         """
1532         #Get used VLAN IDs
1533         usedVlanIDs = []
1534         networks = self.get_network_list()
1535         for net in networks:
1536             if net.get('provider:segmentation_id'):
1537                 usedVlanIDs.append(net.get('provider:segmentation_id'))
1538         used_vlanIDs = set(usedVlanIDs)
1539
1540         #find unused VLAN ID
1541         for vlanID_range in self.config.get('dataplane_net_vlan_range'):
1542             try:
1543                 start_vlanid , end_vlanid = map(int, vlanID_range.replace(" ", "").split("-"))
1544                 for vlanID in xrange(start_vlanid, end_vlanid + 1):
1545                     if vlanID not in used_vlanIDs:
1546                         return vlanID
1547             except Exception as exp:
1548                 raise vimconn.vimconnException("Exception {} occurred while generating VLAN ID.".format(exp))
1549         else:
1550             raise vimconn.vimconnConflictException("Unable to create the SRIOV VLAN network."\
1551                 " All given Vlan IDs {} are in use.".format(self.config.get('dataplane_net_vlan_range')))
1552
1553
1554     def _validate_vlan_ranges(self, dataplane_net_vlan_range):
1555         """
1556         Method to validate user given vlanID ranges
1557             Args:  None
1558             Returns: None
1559         """
1560         for vlanID_range in dataplane_net_vlan_range:
1561             vlan_range = vlanID_range.replace(" ", "")
1562             #validate format
1563             vlanID_pattern = r'(\d)*-(\d)*$'
1564             match_obj = re.match(vlanID_pattern, vlan_range)
1565             if not match_obj:
1566                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}.You must provide "\
1567                 "'dataplane_net_vlan_range' in format [start_ID - end_ID].".format(vlanID_range))
1568
1569             start_vlanid , end_vlanid = map(int,vlan_range.split("-"))
1570             if start_vlanid <= 0 :
1571                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1572                 "Start ID can not be zero. For VLAN "\
1573                 "networks valid IDs are 1 to 4094 ".format(vlanID_range))
1574             if end_vlanid > 4094 :
1575                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1576                 "End VLAN ID can not be greater than 4094. For VLAN "\
1577                 "networks valid IDs are 1 to 4094 ".format(vlanID_range))
1578
1579             if start_vlanid > end_vlanid:
1580                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1581                     "You must provide a 'dataplane_net_vlan_range' in format start_ID - end_ID and "\
1582                     "start_ID < end_ID ".format(vlanID_range))
1583
1584 #NOT USED FUNCTIONS
1585
1586     def new_external_port(self, port_data):
1587         #TODO openstack if needed
1588         '''Adds a external port to VIM'''
1589         '''Returns the port identifier'''
1590         return -vimconn.HTTP_Internal_Server_Error, "osconnector.new_external_port() not implemented"
1591
1592     def connect_port_network(self, port_id, network_id, admin=False):
1593         #TODO openstack if needed
1594         '''Connects a external port to a network'''
1595         '''Returns status code of the VIM response'''
1596         return -vimconn.HTTP_Internal_Server_Error, "osconnector.connect_port_network() not implemented"
1597
1598     def new_user(self, user_name, user_passwd, tenant_id=None):
1599         '''Adds a new user to openstack VIM'''
1600         '''Returns the user identifier'''
1601         self.logger.debug("osconnector: Adding a new user to VIM")
1602         try:
1603             self._reload_connection()
1604             user=self.keystone.users.create(user_name, user_passwd, tenant_id=tenant_id)
1605             #self.keystone.tenants.add_user(self.k_creds["username"], #role)
1606             return user.id
1607         except ksExceptions.ConnectionError as e:
1608             error_value=-vimconn.HTTP_Bad_Request
1609             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1610         except ksExceptions.ClientException as e: #TODO remove
1611             error_value=-vimconn.HTTP_Bad_Request
1612             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1613         #TODO insert exception vimconn.HTTP_Unauthorized
1614         #if reaching here is because an exception
1615         self.logger.debug("new_user " + error_text)
1616         return error_value, error_text
1617
1618     def delete_user(self, user_id):
1619         '''Delete a user from openstack VIM'''
1620         '''Returns the user identifier'''
1621         if self.debug:
1622             print("osconnector: Deleting  a  user from VIM")
1623         try:
1624             self._reload_connection()
1625             self.keystone.users.delete(user_id)
1626             return 1, user_id
1627         except ksExceptions.ConnectionError as e:
1628             error_value=-vimconn.HTTP_Bad_Request
1629             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1630         except ksExceptions.NotFound as e:
1631             error_value=-vimconn.HTTP_Not_Found
1632             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1633         except ksExceptions.ClientException as e: #TODO remove
1634             error_value=-vimconn.HTTP_Bad_Request
1635             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1636         #TODO insert exception vimconn.HTTP_Unauthorized
1637         #if reaching here is because an exception
1638             self.logger.debug("delete_tenant " + error_text)
1639         return error_value, error_text
1640
1641     def get_hosts_info(self):
1642         '''Get the information of deployed hosts
1643         Returns the hosts content'''
1644         if self.debug:
1645             print("osconnector: Getting Host info from VIM")
1646         try:
1647             h_list=[]
1648             self._reload_connection()
1649             hypervisors = self.nova.hypervisors.list()
1650             for hype in hypervisors:
1651                 h_list.append( hype.to_dict() )
1652             return 1, {"hosts":h_list}
1653         except nvExceptions.NotFound as e:
1654             error_value=-vimconn.HTTP_Not_Found
1655             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1656         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1657             error_value=-vimconn.HTTP_Bad_Request
1658             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1659         #TODO insert exception vimconn.HTTP_Unauthorized
1660         #if reaching here is because an exception
1661         self.logger.debug("get_hosts_info " + error_text)
1662         return error_value, error_text
1663
1664     def get_hosts(self, vim_tenant):
1665         '''Get the hosts and deployed instances
1666         Returns the hosts content'''
1667         r, hype_dict = self.get_hosts_info()
1668         if r<0:
1669             return r, hype_dict
1670         hypervisors = hype_dict["hosts"]
1671         try:
1672             servers = self.nova.servers.list()
1673             for hype in hypervisors:
1674                 for server in servers:
1675                     if server.to_dict()['OS-EXT-SRV-ATTR:hypervisor_hostname']==hype['hypervisor_hostname']:
1676                         if 'vm' in hype:
1677                             hype['vm'].append(server.id)
1678                         else:
1679                             hype['vm'] = [server.id]
1680             return 1, hype_dict
1681         except nvExceptions.NotFound as e:
1682             error_value=-vimconn.HTTP_Not_Found
1683             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1684         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1685             error_value=-vimconn.HTTP_Bad_Request
1686             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1687         #TODO insert exception vimconn.HTTP_Unauthorized
1688         #if reaching here is because an exception
1689         self.logger.debug("get_hosts " + error_text)
1690         return error_value, error_text
1691
1692     def new_classification(self, name, ctype, definition):
1693         self.logger.debug(
1694             'Adding a new (Traffic) Classification to VIM, named %s', name)
1695         try:
1696             new_class = None
1697             self._reload_connection()
1698             if ctype not in supportedClassificationTypes:
1699                 raise vimconn.vimconnNotSupportedException(
1700                         'OpenStack VIM connector doesn\'t support provided '
1701                         'Classification Type {}, supported ones are: '
1702                         '{}'.format(ctype, supportedClassificationTypes))
1703             if not self._validate_classification(ctype, definition):
1704                 raise vimconn.vimconnException(
1705                     'Incorrect Classification definition '
1706                     'for the type specified.')
1707             classification_dict = definition
1708             classification_dict['name'] = name
1709
1710             new_class = self.neutron.create_sfc_flow_classifier(
1711                 {'flow_classifier': classification_dict})
1712             return new_class['flow_classifier']['id']
1713         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1714                 neExceptions.NeutronException, ConnectionError) as e:
1715             self.logger.error(
1716                 'Creation of Classification failed.')
1717             self._format_exception(e)
1718
1719     def get_classification(self, class_id):
1720         self.logger.debug(" Getting Classification %s from VIM", class_id)
1721         filter_dict = {"id": class_id}
1722         class_list = self.get_classification_list(filter_dict)
1723         if len(class_list) == 0:
1724             raise vimconn.vimconnNotFoundException(
1725                 "Classification '{}' not found".format(class_id))
1726         elif len(class_list) > 1:
1727             raise vimconn.vimconnConflictException(
1728                 "Found more than one Classification with this criteria")
1729         classification = class_list[0]
1730         return classification
1731
1732     def get_classification_list(self, filter_dict={}):
1733         self.logger.debug("Getting Classifications from VIM filter: '%s'",
1734                           str(filter_dict))
1735         try:
1736             filter_dict_os = filter_dict.copy()
1737             self._reload_connection()
1738             if self.api_version3 and "tenant_id" in filter_dict_os:
1739                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1740             classification_dict = self.neutron.list_sfc_flow_classifiers(
1741                 **filter_dict_os)
1742             classification_list = classification_dict["flow_classifiers"]
1743             self.__classification_os2mano(classification_list)
1744             return classification_list
1745         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1746                 neExceptions.NeutronException, ConnectionError) as e:
1747             self._format_exception(e)
1748
1749     def delete_classification(self, class_id):
1750         self.logger.debug("Deleting Classification '%s' from VIM", class_id)
1751         try:
1752             self._reload_connection()
1753             self.neutron.delete_sfc_flow_classifier(class_id)
1754             return class_id
1755         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1756                 ksExceptions.ClientException, neExceptions.NeutronException,
1757                 ConnectionError) as e:
1758             self._format_exception(e)
1759
1760     def new_sfi(self, name, ingress_ports, egress_ports, sfc_encap=True):
1761         self.logger.debug(
1762             "Adding a new Service Function Instance to VIM, named '%s'", name)
1763         try:
1764             new_sfi = None
1765             self._reload_connection()
1766             correlation = None
1767             if sfc_encap:
1768                 correlation = 'nsh'
1769             if len(ingress_ports) != 1:
1770                 raise vimconn.vimconnNotSupportedException(
1771                     "OpenStack VIM connector can only have "
1772                     "1 ingress port per SFI")
1773             if len(egress_ports) != 1:
1774                 raise vimconn.vimconnNotSupportedException(
1775                     "OpenStack VIM connector can only have "
1776                     "1 egress port per SFI")
1777             sfi_dict = {'name': name,
1778                         'ingress': ingress_ports[0],
1779                         'egress': egress_ports[0],
1780                         'service_function_parameters': {
1781                             'correlation': correlation}}
1782             new_sfi = self.neutron.create_sfc_port_pair({'port_pair': sfi_dict})
1783             return new_sfi['port_pair']['id']
1784         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1785                 neExceptions.NeutronException, ConnectionError) as e:
1786             if new_sfi:
1787                 try:
1788                     self.neutron.delete_sfc_port_pair(
1789                         new_sfi['port_pair']['id'])
1790                 except Exception:
1791                     self.logger.error(
1792                         'Creation of Service Function Instance failed, with '
1793                         'subsequent deletion failure as well.')
1794             self._format_exception(e)
1795
1796     def get_sfi(self, sfi_id):
1797         self.logger.debug(
1798             'Getting Service Function Instance %s from VIM', sfi_id)
1799         filter_dict = {"id": sfi_id}
1800         sfi_list = self.get_sfi_list(filter_dict)
1801         if len(sfi_list) == 0:
1802             raise vimconn.vimconnNotFoundException(
1803                 "Service Function Instance '{}' not found".format(sfi_id))
1804         elif len(sfi_list) > 1:
1805             raise vimconn.vimconnConflictException(
1806                 'Found more than one Service Function Instance '
1807                 'with this criteria')
1808         sfi = sfi_list[0]
1809         return sfi
1810
1811     def get_sfi_list(self, filter_dict={}):
1812         self.logger.debug("Getting Service Function Instances from "
1813                           "VIM filter: '%s'", str(filter_dict))
1814         try:
1815             self._reload_connection()
1816             filter_dict_os = filter_dict.copy()
1817             if self.api_version3 and "tenant_id" in filter_dict_os:
1818                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1819             sfi_dict = self.neutron.list_sfc_port_pairs(**filter_dict_os)
1820             sfi_list = sfi_dict["port_pairs"]
1821             self.__sfi_os2mano(sfi_list)
1822             return sfi_list
1823         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1824                 neExceptions.NeutronException, ConnectionError) as e:
1825             self._format_exception(e)
1826
1827     def delete_sfi(self, sfi_id):
1828         self.logger.debug("Deleting Service Function Instance '%s' "
1829                           "from VIM", sfi_id)
1830         try:
1831             self._reload_connection()
1832             self.neutron.delete_sfc_port_pair(sfi_id)
1833             return sfi_id
1834         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1835                 ksExceptions.ClientException, neExceptions.NeutronException,
1836                 ConnectionError) as e:
1837             self._format_exception(e)
1838
1839     def new_sf(self, name, sfis, sfc_encap=True):
1840         self.logger.debug("Adding a new Service Function to VIM, "
1841                           "named '%s'", name)
1842         try:
1843             new_sf = None
1844             self._reload_connection()
1845             # correlation = None
1846             # if sfc_encap:
1847             #     correlation = 'nsh'
1848             for instance in sfis:
1849                 sfi = self.get_sfi(instance)
1850                 if sfi.get('sfc_encap') != sfc_encap:
1851                     raise vimconn.vimconnNotSupportedException(
1852                         "OpenStack VIM connector requires all SFIs of the "
1853                         "same SF to share the same SFC Encapsulation")
1854             sf_dict = {'name': name,
1855                        'port_pairs': sfis}
1856             new_sf = self.neutron.create_sfc_port_pair_group({
1857                 'port_pair_group': sf_dict})
1858             return new_sf['port_pair_group']['id']
1859         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1860                 neExceptions.NeutronException, ConnectionError) as e:
1861             if new_sf:
1862                 try:
1863                     self.neutron.delete_sfc_port_pair_group(
1864                         new_sf['port_pair_group']['id'])
1865                 except Exception:
1866                     self.logger.error(
1867                         'Creation of Service Function failed, with '
1868                         'subsequent deletion failure as well.')
1869             self._format_exception(e)
1870
1871     def get_sf(self, sf_id):
1872         self.logger.debug("Getting Service Function %s from VIM", sf_id)
1873         filter_dict = {"id": sf_id}
1874         sf_list = self.get_sf_list(filter_dict)
1875         if len(sf_list) == 0:
1876             raise vimconn.vimconnNotFoundException(
1877                 "Service Function '{}' not found".format(sf_id))
1878         elif len(sf_list) > 1:
1879             raise vimconn.vimconnConflictException(
1880                 "Found more than one Service Function with this criteria")
1881         sf = sf_list[0]
1882         return sf
1883
1884     def get_sf_list(self, filter_dict={}):
1885         self.logger.debug("Getting Service Function from VIM filter: '%s'",
1886                           str(filter_dict))
1887         try:
1888             self._reload_connection()
1889             filter_dict_os = filter_dict.copy()
1890             if self.api_version3 and "tenant_id" in filter_dict_os:
1891                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1892             sf_dict = self.neutron.list_sfc_port_pair_groups(**filter_dict_os)
1893             sf_list = sf_dict["port_pair_groups"]
1894             self.__sf_os2mano(sf_list)
1895             return sf_list
1896         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1897                 neExceptions.NeutronException, ConnectionError) as e:
1898             self._format_exception(e)
1899
1900     def delete_sf(self, sf_id):
1901         self.logger.debug("Deleting Service Function '%s' from VIM", sf_id)
1902         try:
1903             self._reload_connection()
1904             self.neutron.delete_sfc_port_pair_group(sf_id)
1905             return sf_id
1906         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1907                 ksExceptions.ClientException, neExceptions.NeutronException,
1908                 ConnectionError) as e:
1909             self._format_exception(e)
1910
1911     def new_sfp(self, name, classifications, sfs, sfc_encap=True, spi=None):
1912         self.logger.debug("Adding a new Service Function Path to VIM, "
1913                           "named '%s'", name)
1914         try:
1915             new_sfp = None
1916             self._reload_connection()
1917             # In networking-sfc the MPLS encapsulation is legacy
1918             # should be used when no full SFC Encapsulation is intended
1919             sfc_encap = 'mpls'
1920             if sfc_encap:
1921                 correlation = 'nsh'
1922             sfp_dict = {'name': name,
1923                         'flow_classifiers': classifications,
1924                         'port_pair_groups': sfs,
1925                         'chain_parameters': {'correlation': correlation}}
1926             if spi:
1927                 sfp_dict['chain_id'] = spi
1928             new_sfp = self.neutron.create_sfc_port_chain({'port_chain': sfp_dict})
1929             return new_sfp["port_chain"]["id"]
1930         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1931                 neExceptions.NeutronException, ConnectionError) as e:
1932             if new_sfp:
1933                 try:
1934                     self.neutron.delete_sfc_port_chain(new_sfp['port_chain']['id'])
1935                 except Exception:
1936                     self.logger.error(
1937                         'Creation of Service Function Path failed, with '
1938                         'subsequent deletion failure as well.')
1939             self._format_exception(e)
1940
1941     def get_sfp(self, sfp_id):
1942         self.logger.debug(" Getting Service Function Path %s from VIM", sfp_id)
1943         filter_dict = {"id": sfp_id}
1944         sfp_list = self.get_sfp_list(filter_dict)
1945         if len(sfp_list) == 0:
1946             raise vimconn.vimconnNotFoundException(
1947                 "Service Function Path '{}' not found".format(sfp_id))
1948         elif len(sfp_list) > 1:
1949             raise vimconn.vimconnConflictException(
1950                 "Found more than one Service Function Path with this criteria")
1951         sfp = sfp_list[0]
1952         return sfp
1953
1954     def get_sfp_list(self, filter_dict={}):
1955         self.logger.debug("Getting Service Function Paths from VIM filter: "
1956                           "'%s'", str(filter_dict))
1957         try:
1958             self._reload_connection()
1959             filter_dict_os = filter_dict.copy()
1960             if self.api_version3 and "tenant_id" in filter_dict_os:
1961                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1962             sfp_dict = self.neutron.list_sfc_port_chains(**filter_dict_os)
1963             sfp_list = sfp_dict["port_chains"]
1964             self.__sfp_os2mano(sfp_list)
1965             return sfp_list
1966         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1967                 neExceptions.NeutronException, ConnectionError) as e:
1968             self._format_exception(e)
1969
1970     def delete_sfp(self, sfp_id):
1971         self.logger.debug(
1972             "Deleting Service Function Path '%s' from VIM", sfp_id)
1973         try:
1974             self._reload_connection()
1975             self.neutron.delete_sfc_port_chain(sfp_id)
1976             return sfp_id
1977         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1978                 ksExceptions.ClientException, neExceptions.NeutronException,
1979                 ConnectionError) as e:
1980             self._format_exception(e)