osm_ro/vimconn_openstack.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 osconnector implements all the methods to interact with openstack using the python-neutronclient.
  26
  27 For the VNF forwarding graph, The OpenStack VIM connector calls the
  28 networking-sfc Neutron extension methods, whose resources are mapped
  29 to the VIM connector's SFC resources as follows:
  30 - Classification (OSM) -> Flow Classifier (Neutron)
  31 - Service Function Instance (OSM) -> Port Pair (Neutron)
  32 - Service Function (OSM) -> Port Pair Group (Neutron)
  33 - Service Function Path (OSM) -> Port Chain (Neutron)
  34 '''
  35 __author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes, xFlow Research, Igor D.C., Eduardo Sousa"
  36 __date__  = "$22-sep-2017 23:59:59$"
  37
  38 import vimconn
  39 # import json
  40 import logging
  41 import netaddr
  42 import time
  43 import yaml
  44 import random
  45 import re
  46 import copy
  47
  48 from novaclient import client as nClient, exceptions as nvExceptions
  49 from keystoneauth1.identity import v2, v3
  50 from keystoneauth1 import session
  51 import keystoneclient.exceptions as ksExceptions
  52 import keystoneclient.v3.client as ksClient_v3
  53 import keystoneclient.v2_0.client as ksClient_v2
  54 from glanceclient import client as glClient
  55 import glanceclient.exc as gl1Exceptions
  56 from  cinderclient import client as cClient
  57 from httplib import HTTPException
  58 from neutronclient.neutron import client as neClient
  59 from neutronclient.common import exceptions as neExceptions
  60 from requests.exceptions import ConnectionError
  61
  62
  63 """contain the openstack virtual machine status to openmano status"""
  64 vmStatus2manoFormat={'ACTIVE':'ACTIVE',
  65                      'PAUSED':'PAUSED',
  66                      'SUSPENDED': 'SUSPENDED',
  67                      'SHUTOFF':'INACTIVE',
  68                      'BUILD':'BUILD',
  69                      'ERROR':'ERROR','DELETED':'DELETED'
  70                      }
  71 netStatus2manoFormat={'ACTIVE':'ACTIVE','PAUSED':'PAUSED','INACTIVE':'INACTIVE','BUILD':'BUILD','ERROR':'ERROR','DELETED':'DELETED'
  72                      }
  73
  74 supportedClassificationTypes = ['legacy_flow_classifier']
  75
  76 #global var to have a timeout creating and deleting volumes
  77 volume_timeout = 600
  78 server_timeout = 600
  79
  80 class vimconnector(vimconn.vimconnector):
  81     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None,
  82                  log_level=None, config={}, persistent_info={}):
  83         '''using common constructor parameters. In this case
  84         'url' is the keystone authorization url,
  85         'url_admin' is not use
  86         '''
  87         api_version = config.get('APIversion')
  88         if api_version and api_version not in ('v3.3', 'v2.0', '2', '3'):
  89             raise vimconn.vimconnException("Invalid value '{}' for config:APIversion. "
  90                                            "Allowed values are 'v3.3', 'v2.0', '2' or '3'".format(api_version))
  91         vim_type = config.get('vim_type')
  92         if vim_type and vim_type not in ('vio', 'VIO'):
  93             raise vimconn.vimconnException("Invalid value '{}' for config:vim_type."
  94                             "Allowed values are 'vio' or 'VIO'".format(vim_type))
  95
  96         if config.get('dataplane_net_vlan_range') is not None:
  97             #validate vlan ranges provided by user
  98             self._validate_vlan_ranges(config.get('dataplane_net_vlan_range'))
  99
 100         vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level,
 101                                       config)
 102
 103         if self.config.get("insecure") and self.config.get("ca_cert"):
 104             raise vimconn.vimconnException("options insecure and ca_cert are mutually exclusive")
 105         self.verify = True
 106         if self.config.get("insecure"):
 107             self.verify = False
 108         if self.config.get("ca_cert"):
 109             self.verify = self.config.get("ca_cert")
 110
 111         if not url:
 112             raise TypeError('url param can not be NoneType')
 113         self.persistent_info = persistent_info
 114         self.availability_zone = persistent_info.get('availability_zone', None)
 115         self.session = persistent_info.get('session', {'reload_client': True})
 116         self.nova = self.session.get('nova')
 117         self.neutron = self.session.get('neutron')
 118         self.cinder = self.session.get('cinder')
 119         self.glance = self.session.get('glance')
 120         # self.glancev1 = self.session.get('glancev1')
 121         self.keystone = self.session.get('keystone')
 122         self.api_version3 = self.session.get('api_version3')
 123         self.vim_type = self.config.get("vim_type")
 124         if self.vim_type:
 125             self.vim_type = self.vim_type.upper()
 126         if self.config.get("use_internal_endpoint"):
 127             self.endpoint_type = "internalURL"
 128         else:
 129             self.endpoint_type = None
 130
 131         self.logger = logging.getLogger('openmano.vim.openstack')
 132
 133         ####### VIO Specific Changes #########
 134         if self.vim_type == "VIO":
 135             self.logger = logging.getLogger('openmano.vim.vio')
 136
 137         if log_level:
 138             self.logger.setLevel( getattr(logging, log_level))
 139
 140     def __getitem__(self, index):
 141         """Get individuals parameters.
 142         Throw KeyError"""
 143         if index == 'project_domain_id':
 144             return self.config.get("project_domain_id")
 145         elif index == 'user_domain_id':
 146             return self.config.get("user_domain_id")
 147         else:
 148             return vimconn.vimconnector.__getitem__(self, index)
 149
 150     def __setitem__(self, index, value):
 151         """Set individuals parameters and it is marked as dirty so to force connection reload.
 152         Throw KeyError"""
 153         if index == 'project_domain_id':
 154             self.config["project_domain_id"] = value
 155         elif index == 'user_domain_id':
 156                 self.config["user_domain_id"] = value
 157         else:
 158             vimconn.vimconnector.__setitem__(self, index, value)
 159         self.session['reload_client'] = True
 160
 161     def _reload_connection(self):
 162         '''Called before any operation, it check if credentials has changed
 163         Throw keystoneclient.apiclient.exceptions.AuthorizationFailure
 164         '''
 165         #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-)
 166         if self.session['reload_client']:
 167             if self.config.get('APIversion'):
 168                 self.api_version3 = self.config['APIversion'] == 'v3.3' or self.config['APIversion'] == '3'
 169             else:  # get from ending auth_url that end with v3 or with v2.0
 170                 self.api_version3 =  self.url.endswith("/v3") or self.url.endswith("/v3/")
 171             self.session['api_version3'] = self.api_version3
 172             if self.api_version3:
 173                 if self.config.get('project_domain_id') or self.config.get('project_domain_name'):
 174                     project_domain_id_default = None
 175                 else:
 176                     project_domain_id_default = 'default'
 177                 if self.config.get('user_domain_id') or self.config.get('user_domain_name'):
 178                     user_domain_id_default = None
 179                 else:
 180                     user_domain_id_default = 'default'
 181                 auth = v3.Password(auth_url=self.url,
 182                                    username=self.user,
 183                                    password=self.passwd,
 184                                    project_name=self.tenant_name,
 185                                    project_id=self.tenant_id,
 186                                    project_domain_id=self.config.get('project_domain_id', project_domain_id_default),
 187                                    user_domain_id=self.config.get('user_domain_id', user_domain_id_default),
 188                                    project_domain_name=self.config.get('project_domain_name'),
 189                                    user_domain_name=self.config.get('user_domain_name'))
 190             else:
 191                 auth = v2.Password(auth_url=self.url,
 192                                    username=self.user,
 193                                    password=self.passwd,
 194                                    tenant_name=self.tenant_name,
 195                                    tenant_id=self.tenant_id)
 196             sess = session.Session(auth=auth, verify=self.verify)
 197             if self.api_version3:
 198                 self.keystone = ksClient_v3.Client(session=sess, endpoint_type=self.endpoint_type)
 199             else:
 200                 self.keystone = ksClient_v2.Client(session=sess, endpoint_type=self.endpoint_type)
 201             self.session['keystone'] = self.keystone
 202             # In order to enable microversion functionality an explicit microversion must be specified in 'config'.
 203             # This implementation approach is due to the warning message in
 204             # https://developer.openstack.org/api-guide/compute/microversions.html
 205             # where it is stated that microversion backwards compatibility is not guaranteed and clients should
 206             # always require an specific microversion.
 207             # To be able to use 'device role tagging' functionality define 'microversion: 2.32' in datacenter config
 208             version = self.config.get("microversion")
 209             if not version:
 210                 version = "2.1"
 211             self.nova = self.session['nova'] = nClient.Client(str(version), session=sess, endpoint_type=self.endpoint_type)
 212             self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess, endpoint_type=self.endpoint_type)
 213             self.cinder = self.session['cinder'] = cClient.Client(2, session=sess, endpoint_type=self.endpoint_type)
 214             if self.endpoint_type == "internalURL":
 215                 glance_service_id = self.keystone.services.list(name="glance")[0].id
 216                 glance_endpoint = self.keystone.endpoints.list(glance_service_id, interface="internal")[0].url
 217             else:
 218                 glance_endpoint = None
 219             self.glance = self.session['glance'] = glClient.Client(2, session=sess, endpoint=glance_endpoint)
 220             #using version 1 of glance client in new_image()
 221             # self.glancev1 = self.session['glancev1'] = glClient.Client('1', session=sess,
 222             #                                                            endpoint=glance_endpoint)
 223             self.session['reload_client'] = False
 224             self.persistent_info['session'] = self.session
 225             # add availablity zone info inside  self.persistent_info
 226             self._set_availablity_zones()
 227             self.persistent_info['availability_zone'] = self.availability_zone
 228
 229     def __net_os2mano(self, net_list_dict):
 230         '''Transform the net openstack format to mano format
 231         net_list_dict can be a list of dict or a single dict'''
 232         if type(net_list_dict) is dict:
 233             net_list_=(net_list_dict,)
 234         elif type(net_list_dict) is list:
 235             net_list_=net_list_dict
 236         else:
 237             raise TypeError("param net_list_dict must be a list or a dictionary")
 238         for net in net_list_:
 239             if net.get('provider:network_type') == "vlan":
 240                 net['type']='data'
 241             else:
 242                 net['type']='bridge'
 243
 244     def __classification_os2mano(self, class_list_dict):
 245         """Transform the openstack format (Flow Classifier) to mano format
 246         (Classification) class_list_dict can be a list of dict or a single dict
 247         """
 248         if isinstance(class_list_dict, dict):
 249             class_list_ = [class_list_dict]
 250         elif isinstance(class_list_dict, list):
 251             class_list_ = class_list_dict
 252         else:
 253             raise TypeError(
 254                 "param class_list_dict must be a list or a dictionary")
 255         for classification in class_list_:
 256             id = classification.pop('id')
 257             name = classification.pop('name')
 258             description = classification.pop('description')
 259             project_id = classification.pop('project_id')
 260             tenant_id = classification.pop('tenant_id')
 261             original_classification = copy.deepcopy(classification)
 262             classification.clear()
 263             classification['ctype'] = 'legacy_flow_classifier'
 264             classification['definition'] = original_classification
 265             classification['id'] = id
 266             classification['name'] = name
 267             classification['description'] = description
 268             classification['project_id'] = project_id
 269             classification['tenant_id'] = tenant_id
 270
 271     def __sfi_os2mano(self, sfi_list_dict):
 272         """Transform the openstack format (Port Pair) to mano format (SFI)
 273         sfi_list_dict can be a list of dict or a single dict
 274         """
 275         if isinstance(sfi_list_dict, dict):
 276             sfi_list_ = [sfi_list_dict]
 277         elif isinstance(sfi_list_dict, list):
 278             sfi_list_ = sfi_list_dict
 279         else:
 280             raise TypeError(
 281                 "param sfi_list_dict must be a list or a dictionary")
 282         for sfi in sfi_list_:
 283             sfi['ingress_ports'] = []
 284             sfi['egress_ports'] = []
 285             if sfi.get('ingress'):
 286                 sfi['ingress_ports'].append(sfi['ingress'])
 287             if sfi.get('egress'):
 288                 sfi['egress_ports'].append(sfi['egress'])
 289             del sfi['ingress']
 290             del sfi['egress']
 291             params = sfi.get('service_function_parameters')
 292             sfc_encap = False
 293             if params:
 294                 correlation = params.get('correlation')
 295                 if correlation:
 296                     sfc_encap = True
 297             sfi['sfc_encap'] = sfc_encap
 298             del sfi['service_function_parameters']
 299
 300     def __sf_os2mano(self, sf_list_dict):
 301         """Transform the openstack format (Port Pair Group) to mano format (SF)
 302         sf_list_dict can be a list of dict or a single dict
 303         """
 304         if isinstance(sf_list_dict, dict):
 305             sf_list_ = [sf_list_dict]
 306         elif isinstance(sf_list_dict, list):
 307             sf_list_ = sf_list_dict
 308         else:
 309             raise TypeError(
 310                 "param sf_list_dict must be a list or a dictionary")
 311         for sf in sf_list_:
 312             del sf['port_pair_group_parameters']
 313             sf['sfis'] = sf['port_pairs']
 314             del sf['port_pairs']
 315
 316     def __sfp_os2mano(self, sfp_list_dict):
 317         """Transform the openstack format (Port Chain) to mano format (SFP)
 318         sfp_list_dict can be a list of dict or a single dict
 319         """
 320         if isinstance(sfp_list_dict, dict):
 321             sfp_list_ = [sfp_list_dict]
 322         elif isinstance(sfp_list_dict, list):
 323             sfp_list_ = sfp_list_dict
 324         else:
 325             raise TypeError(
 326                 "param sfp_list_dict must be a list or a dictionary")
 327         for sfp in sfp_list_:
 328             params = sfp.pop('chain_parameters')
 329             sfc_encap = False
 330             if params:
 331                 correlation = params.get('correlation')
 332                 if correlation:
 333                     sfc_encap = True
 334             sfp['sfc_encap'] = sfc_encap
 335             sfp['spi'] = sfp.pop('chain_id')
 336             sfp['classifications'] = sfp.pop('flow_classifiers')
 337             sfp['service_functions'] = sfp.pop('port_pair_groups')
 338
 339     # placeholder for now; read TODO note below
 340     def _validate_classification(self, type, definition):
 341         # only legacy_flow_classifier Type is supported at this point
 342         return True
 343         # TODO(igordcard): this method should be an abstract method of an
 344         # abstract Classification class to be implemented by the specific
 345         # Types. Also, abstract vimconnector should call the validation
 346         # method before the implemented VIM connectors are called.
 347
 348     def _format_exception(self, exception):
 349         '''Transform a keystone, nova, neutron  exception into a vimconn exception'''
 350         if isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError,
 351                                   ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed
 352                                   )):
 353             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 354         elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException,
 355                                     neExceptions.NeutronException, nvExceptions.BadRequest)):
 356             raise vimconn.vimconnUnexpectedResponse(type(exception).__name__ + ": " + str(exception))
 357         elif isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound)):
 358             raise vimconn.vimconnNotFoundException(type(exception).__name__ + ": " + str(exception))
 359         elif isinstance(exception, nvExceptions.Conflict):
 360             raise vimconn.vimconnConflictException(type(exception).__name__ + ": " + str(exception))
 361         elif isinstance(exception, vimconn.vimconnException):
 362             raise exception
 363         else:  # ()
 364             self.logger.error("General Exception " + str(exception), exc_info=True)
 365             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 366
 367     def get_tenant_list(self, filter_dict={}):
 368         '''Obtain tenants of VIM
 369         filter_dict can contain the following keys:
 370             name: filter by tenant name
 371             id: filter by tenant uuid/id
 372             <other VIM specific>
 373         Returns the tenant list of dictionaries: [{'name':'<name>, 'id':'<id>, ...}, ...]
 374         '''
 375         self.logger.debug("Getting tenants from VIM filter: '%s'", str(filter_dict))
 376         try:
 377             self._reload_connection()
 378             if self.api_version3:
 379                 project_class_list = self.keystone.projects.list(name=filter_dict.get("name"))
 380             else:
 381                 project_class_list = self.keystone.tenants.findall(**filter_dict)
 382             project_list=[]
 383             for project in project_class_list:
 384                 if filter_dict.get('id') and filter_dict["id"] != project.id:
 385                     continue
 386                 project_list.append(project.to_dict())
 387             return project_list
 388         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError) as e:
 389             self._format_exception(e)
 390
 391     def new_tenant(self, tenant_name, tenant_description):
 392         '''Adds a new tenant to openstack VIM. Returns the tenant identifier'''
 393         self.logger.debug("Adding a new tenant name: %s", tenant_name)
 394         try:
 395             self._reload_connection()
 396             if self.api_version3:
 397                 project = self.keystone.projects.create(tenant_name, self.config.get("project_domain_id", "default"),
 398                                                         description=tenant_description, is_domain=False)
 399             else:
 400                 project = self.keystone.tenants.create(tenant_name, tenant_description)
 401             return project.id
 402         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 403             self._format_exception(e)
 404
 405     def delete_tenant(self, tenant_id):
 406         '''Delete a tenant from openstack VIM. Returns the old tenant identifier'''
 407         self.logger.debug("Deleting tenant %s from VIM", tenant_id)
 408         try:
 409             self._reload_connection()
 410             if self.api_version3:
 411                 self.keystone.projects.delete(tenant_id)
 412             else:
 413                 self.keystone.tenants.delete(tenant_id)
 414             return tenant_id
 415         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 416             self._format_exception(e)
 417
 418     def new_network(self,net_name, net_type, ip_profile=None, shared=False, vlan=None):
 419         '''Adds a tenant network to VIM. Returns the network identifier'''
 420         self.logger.debug("Adding a new network to VIM name '%s', type '%s'", net_name, net_type)
 421         #self.logger.debug(">>>>>>>>>>>>>>>>>> IP profile %s", str(ip_profile))
 422         try:
 423             new_net = None
 424             self._reload_connection()
 425             network_dict = {'name': net_name, 'admin_state_up': True}
 426             if net_type=="data" or net_type=="ptp":
 427                 if self.config.get('dataplane_physical_net') == None:
 428                     raise vimconn.vimconnConflictException("You must provide a 'dataplane_physical_net' at config value before creating sriov network")
 429                 network_dict["provider:physical_network"] = self.config['dataplane_physical_net'] #"physnet_sriov" #TODO physical
 430                 network_dict["provider:network_type"]     = "vlan"
 431                 if vlan!=None:
 432                     network_dict["provider:network_type"] = vlan
 433
 434                 ####### VIO Specific Changes #########
 435                 if self.vim_type == "VIO":
 436                     if vlan is not None:
 437                         network_dict["provider:segmentation_id"] = vlan
 438                     else:
 439                         if self.config.get('dataplane_net_vlan_range') is None:
 440                             raise vimconn.vimconnConflictException("You must provide "\
 441                                 "'dataplane_net_vlan_range' in format [start_ID - end_ID]"\
 442                                 "at config value before creating sriov network with vlan tag")
 443
 444                         network_dict["provider:segmentation_id"] = self._genrate_vlanID()
 445
 446             network_dict["shared"]=shared
 447             new_net=self.neutron.create_network({'network':network_dict})
 448             #print new_net
 449             #create subnetwork, even if there is no profile
 450             if not ip_profile:
 451                 ip_profile = {}
 452             if not ip_profile.get('subnet_address'):
 453                 #Fake subnet is required
 454                 subnet_rand = random.randint(0, 255)
 455                 ip_profile['subnet_address'] = "192.168.{}.0/24".format(subnet_rand)
 456             if 'ip_version' not in ip_profile:
 457                 ip_profile['ip_version'] = "IPv4"
 458             subnet = {"name":net_name+"-subnet",
 459                     "network_id": new_net["network"]["id"],
 460                     "ip_version": 4 if ip_profile['ip_version']=="IPv4" else 6,
 461                     "cidr": ip_profile['subnet_address']
 462                     }
 463             # Gateway should be set to None if not needed. Otherwise openstack assigns one by default
 464             if ip_profile.get('gateway_address'):
 465                 subnet['gateway_ip'] = ip_profile['gateway_address']
 466             else:
 467                 subnet['gateway_ip'] = None
 468             if ip_profile.get('dns_address'):
 469                 subnet['dns_nameservers'] = ip_profile['dns_address'].split(";")
 470             if 'dhcp_enabled' in ip_profile:
 471                 subnet['enable_dhcp'] = False if \
 472                     ip_profile['dhcp_enabled']=="false" or ip_profile['dhcp_enabled']==False else True
 473             if ip_profile.get('dhcp_start_address'):
 474                 subnet['allocation_pools'] = []
 475                 subnet['allocation_pools'].append(dict())
 476                 subnet['allocation_pools'][0]['start'] = ip_profile['dhcp_start_address']
 477             if ip_profile.get('dhcp_count'):
 478                 #parts = ip_profile['dhcp_start_address'].split('.')
 479                 #ip_int = (int(parts[0]) << 24) + (int(parts[1]) << 16) + (int(parts[2]) << 8) + int(parts[3])
 480                 ip_int = int(netaddr.IPAddress(ip_profile['dhcp_start_address']))
 481                 ip_int += ip_profile['dhcp_count'] - 1
 482                 ip_str = str(netaddr.IPAddress(ip_int))
 483                 subnet['allocation_pools'][0]['end'] = ip_str
 484             #self.logger.debug(">>>>>>>>>>>>>>>>>> Subnet: %s", str(subnet))
 485             self.neutron.create_subnet({"subnet": subnet} )
 486             return new_net["network"]["id"]
 487         except Exception as e:
 488             if new_net:
 489                 self.neutron.delete_network(new_net['network']['id'])
 490             self._format_exception(e)
 491
 492     def get_network_list(self, filter_dict={}):
 493         '''Obtain tenant networks of VIM
 494         Filter_dict can be:
 495             name: network name
 496             id: network uuid
 497             shared: boolean
 498             tenant_id: tenant
 499             admin_state_up: boolean
 500             status: 'ACTIVE'
 501         Returns the network list of dictionaries
 502         '''
 503         self.logger.debug("Getting network from VIM filter: '%s'", str(filter_dict))
 504         try:
 505             self._reload_connection()
 506             filter_dict_os = filter_dict.copy()
 507             if self.api_version3 and "tenant_id" in filter_dict_os:
 508                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')  #T ODO check
 509             net_dict = self.neutron.list_networks(**filter_dict_os)
 510             net_list = net_dict["networks"]
 511             self.__net_os2mano(net_list)
 512             return net_list
 513         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 514             self._format_exception(e)
 515
 516     def get_network(self, net_id):
 517         '''Obtain details of network from VIM
 518         Returns the network information from a network id'''
 519         self.logger.debug(" Getting tenant network %s from VIM", net_id)
 520         filter_dict={"id": net_id}
 521         net_list = self.get_network_list(filter_dict)
 522         if len(net_list)==0:
 523             raise vimconn.vimconnNotFoundException("Network '{}' not found".format(net_id))
 524         elif len(net_list)>1:
 525             raise vimconn.vimconnConflictException("Found more than one network with this criteria")
 526         net = net_list[0]
 527         subnets=[]
 528         for subnet_id in net.get("subnets", () ):
 529             try:
 530                 subnet = self.neutron.show_subnet(subnet_id)
 531             except Exception as e:
 532                 self.logger.error("osconnector.get_network(): Error getting subnet %s %s" % (net_id, str(e)))
 533                 subnet = {"id": subnet_id, "fault": str(e)}
 534             subnets.append(subnet)
 535         net["subnets"] = subnets
 536         net["encapsulation"] = net.get('provider:network_type')
 537         net["segmentation_id"] = net.get('provider:segmentation_id')
 538         return net
 539
 540     def delete_network(self, net_id):
 541         '''Deletes a tenant network from VIM. Returns the old network identifier'''
 542         self.logger.debug("Deleting network '%s' from VIM", net_id)
 543         try:
 544             self._reload_connection()
 545             #delete VM ports attached to this networks before the network
 546             ports = self.neutron.list_ports(network_id=net_id)
 547             for p in ports['ports']:
 548                 try:
 549                     self.neutron.delete_port(p["id"])
 550                 except Exception as e:
 551                     self.logger.error("Error deleting port %s: %s", p["id"], str(e))
 552             self.neutron.delete_network(net_id)
 553             return net_id
 554         except (neExceptions.ConnectionFailed, neExceptions.NetworkNotFoundClient, neExceptions.NeutronException,
 555                 ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 556             self._format_exception(e)
 557
 558     def refresh_nets_status(self, net_list):
 559         '''Get the status of the networks
 560            Params: the list of network identifiers
 561            Returns a dictionary with:
 562                 net_id:         #VIM id of this network
 563                     status:     #Mandatory. Text with one of:
 564                                 #  DELETED (not found at vim)
 565                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 566                                 #  OTHER (Vim reported other status not understood)
 567                                 #  ERROR (VIM indicates an ERROR status)
 568                                 #  ACTIVE, INACTIVE, DOWN (admin down),
 569                                 #  BUILD (on building process)
 570                                 #
 571                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 572                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 573
 574         '''
 575         net_dict={}
 576         for net_id in net_list:
 577             net = {}
 578             try:
 579                 net_vim = self.get_network(net_id)
 580                 if net_vim['status'] in netStatus2manoFormat:
 581                     net["status"] = netStatus2manoFormat[ net_vim['status'] ]
 582                 else:
 583                     net["status"] = "OTHER"
 584                     net["error_msg"] = "VIM status reported " + net_vim['status']
 585
 586                 if net['status'] == "ACTIVE" and not net_vim['admin_state_up']:
 587                     net['status'] = 'DOWN'
 588                 try:
 589                     net['vim_info'] = yaml.safe_dump(net_vim, default_flow_style=True, width=256)
 590                 except yaml.representer.RepresenterError:
 591                     net['vim_info'] = str(net_vim)
 592                 if net_vim.get('fault'):  #TODO
 593                     net['error_msg'] = str(net_vim['fault'])
 594             except vimconn.vimconnNotFoundException as e:
 595                 self.logger.error("Exception getting net status: %s", str(e))
 596                 net['status'] = "DELETED"
 597                 net['error_msg'] = str(e)
 598             except vimconn.vimconnException as e:
 599                 self.logger.error("Exception getting net status: %s", str(e))
 600                 net['status'] = "VIM_ERROR"
 601                 net['error_msg'] = str(e)
 602             net_dict[net_id] = net
 603         return net_dict
 604
 605     def get_flavor(self, flavor_id):
 606         '''Obtain flavor details from the  VIM. Returns the flavor dict details'''
 607         self.logger.debug("Getting flavor '%s'", flavor_id)
 608         try:
 609             self._reload_connection()
 610             flavor = self.nova.flavors.find(id=flavor_id)
 611             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 612             return flavor.to_dict()
 613         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 614             self._format_exception(e)
 615
 616     def get_flavor_id_from_data(self, flavor_dict):
 617         """Obtain flavor id that match the flavor description
 618            Returns the flavor_id or raises a vimconnNotFoundException
 619            flavor_dict: contains the required ram, vcpus, disk
 620            If 'use_existing_flavors' is set to True at config, the closer flavor that provides same or more ram, vcpus
 621                 and disk is returned. Otherwise a flavor with exactly same ram, vcpus and disk is returned or a
 622                 vimconnNotFoundException is raised
 623         """
 624         exact_match = False if self.config.get('use_existing_flavors') else True
 625         try:
 626             self._reload_connection()
 627             flavor_candidate_id = None
 628             flavor_candidate_data = (10000, 10000, 10000)
 629             flavor_target = (flavor_dict["ram"], flavor_dict["vcpus"], flavor_dict["disk"])
 630             # numa=None
 631             numas = flavor_dict.get("extended", {}).get("numas")
 632             if numas:
 633                 #TODO
 634                 raise vimconn.vimconnNotFoundException("Flavor with EPA still not implemted")
 635                 # if len(numas) > 1:
 636                 #     raise vimconn.vimconnNotFoundException("Cannot find any flavor with more than one numa")
 637                 # numa=numas[0]
 638                 # numas = extended.get("numas")
 639             for flavor in self.nova.flavors.list():
 640                 epa = flavor.get_keys()
 641                 if epa:
 642                     continue
 643                     # TODO
 644                 flavor_data = (flavor.ram, flavor.vcpus, flavor.disk)
 645                 if flavor_data == flavor_target:
 646                     return flavor.id
 647                 elif not exact_match and flavor_target < flavor_data < flavor_candidate_data:
 648                     flavor_candidate_id = flavor.id
 649                     flavor_candidate_data = flavor_data
 650             if not exact_match and flavor_candidate_id:
 651                 return flavor_candidate_id
 652             raise vimconn.vimconnNotFoundException("Cannot find any flavor matching '{}'".format(str(flavor_dict)))
 653         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 654             self._format_exception(e)
 655
 656     def new_flavor(self, flavor_data, change_name_if_used=True):
 657         '''Adds a tenant flavor to openstack VIM
 658         if change_name_if_used is True, it will change name in case of conflict, because it is not supported name repetition
 659         Returns the flavor identifier
 660         '''
 661         self.logger.debug("Adding flavor '%s'", str(flavor_data))
 662         retry=0
 663         max_retries=3
 664         name_suffix = 0
 665         name=flavor_data['name']
 666         while retry<max_retries:
 667             retry+=1
 668             try:
 669                 self._reload_connection()
 670                 if change_name_if_used:
 671                     #get used names
 672                     fl_names=[]
 673                     fl=self.nova.flavors.list()
 674                     for f in fl:
 675                         fl_names.append(f.name)
 676                     while name in fl_names:
 677                         name_suffix += 1
 678                         name = flavor_data['name']+"-" + str(name_suffix)
 679
 680                 ram = flavor_data.get('ram',64)
 681                 vcpus = flavor_data.get('vcpus',1)
 682                 numa_properties=None
 683
 684                 extended = flavor_data.get("extended")
 685                 if extended:
 686                     numas=extended.get("numas")
 687                     if numas:
 688                         numa_nodes = len(numas)
 689                         if numa_nodes > 1:
 690                             return -1, "Can not add flavor with more than one numa"
 691                         numa_properties = {"hw:numa_nodes":str(numa_nodes)}
 692                         numa_properties["hw:mem_page_size"] = "large"
 693                         numa_properties["hw:cpu_policy"] = "dedicated"
 694                         numa_properties["hw:numa_mempolicy"] = "strict"
 695                         if self.vim_type == "VIO":
 696                             numa_properties["vmware:extra_config"] = '{"numa.nodeAffinity":"0"}'
 697                             numa_properties["vmware:latency_sensitivity_level"] = "high"
 698                         for numa in numas:
 699                             #overwrite ram and vcpus
 700                             #check if key 'memory' is present in numa else use ram value at flavor
 701                             if 'memory' in numa:
 702                                 ram = numa['memory']*1024
 703                             #See for reference: https://specs.openstack.org/openstack/nova-specs/specs/mitaka/implemented/virt-driver-cpu-thread-pinning.html
 704                             if 'paired-threads' in numa:
 705                                 vcpus = numa['paired-threads']*2
 706                                 #cpu_thread_policy "require" implies that the compute node must have an STM architecture
 707                                 numa_properties["hw:cpu_thread_policy"] = "require"
 708                                 numa_properties["hw:cpu_policy"] = "dedicated"
 709                             elif 'cores' in numa:
 710                                 vcpus = numa['cores']
 711                                 # cpu_thread_policy "prefer" implies that the host must not have an SMT architecture, or a non-SMT architecture will be emulated
 712                                 numa_properties["hw:cpu_thread_policy"] = "isolate"
 713                                 numa_properties["hw:cpu_policy"] = "dedicated"
 714                             elif 'threads' in numa:
 715                                 vcpus = numa['threads']
 716                                 # cpu_thread_policy "prefer" implies that the host may or may not have an SMT architecture
 717                                 numa_properties["hw:cpu_thread_policy"] = "prefer"
 718                                 numa_properties["hw:cpu_policy"] = "dedicated"
 719                             # for interface in numa.get("interfaces",() ):
 720                             #     if interface["dedicated"]=="yes":
 721                             #         raise vimconn.vimconnException("Passthrough interfaces are not supported for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable)
 722                             #     #TODO, add the key 'pci_passthrough:alias"="<label at config>:<number ifaces>"' when a way to connect it is available
 723
 724                 #create flavor
 725                 new_flavor=self.nova.flavors.create(name,
 726                                 ram,
 727                                 vcpus,
 728                                 flavor_data.get('disk',0),
 729                                 is_public=flavor_data.get('is_public', True)
 730                             )
 731                 #add metadata
 732                 if numa_properties:
 733                     new_flavor.set_keys(numa_properties)
 734                 return new_flavor.id
 735             except nvExceptions.Conflict as e:
 736                 if change_name_if_used and retry < max_retries:
 737                     continue
 738                 self._format_exception(e)
 739             #except nvExceptions.BadRequest as e:
 740             except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 741                 self._format_exception(e)
 742
 743     def delete_flavor(self,flavor_id):
 744         '''Deletes a tenant flavor from openstack VIM. Returns the old flavor_id
 745         '''
 746         try:
 747             self._reload_connection()
 748             self.nova.flavors.delete(flavor_id)
 749             return flavor_id
 750         #except nvExceptions.BadRequest as e:
 751         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 752             self._format_exception(e)
 753
 754     def new_image(self,image_dict):
 755         '''
 756         Adds a tenant image to VIM. imge_dict is a dictionary with:
 757             name: name
 758             disk_format: qcow2, vhd, vmdk, raw (by default), ...
 759             location: path or URI
 760             public: "yes" or "no"
 761             metadata: metadata of the image
 762         Returns the image_id
 763         '''
 764         retry=0
 765         max_retries=3
 766         while retry<max_retries:
 767             retry+=1
 768             try:
 769                 self._reload_connection()
 770                 #determine format  http://docs.openstack.org/developer/glance/formats.html
 771                 if "disk_format" in image_dict:
 772                     disk_format=image_dict["disk_format"]
 773                 else: #autodiscover based on extension
 774                     if image_dict['location'].endswith(".qcow2"):
 775                         disk_format="qcow2"
 776                     elif image_dict['location'].endswith(".vhd"):
 777                         disk_format="vhd"
 778                     elif image_dict['location'].endswith(".vmdk"):
 779                         disk_format="vmdk"
 780                     elif image_dict['location'].endswith(".vdi"):
 781                         disk_format="vdi"
 782                     elif image_dict['location'].endswith(".iso"):
 783                         disk_format="iso"
 784                     elif image_dict['location'].endswith(".aki"):
 785                         disk_format="aki"
 786                     elif image_dict['location'].endswith(".ari"):
 787                         disk_format="ari"
 788                     elif image_dict['location'].endswith(".ami"):
 789                         disk_format="ami"
 790                     else:
 791                         disk_format="raw"
 792                 self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location'])
 793                 new_image = self.glance.images.create(name=image_dict['name'])
 794                 if image_dict['location'].startswith("http"):
 795                     # TODO there is not a method to direct download. It must be downloaded locally with requests
 796                     raise vimconn.vimconnNotImplemented("Cannot create image from URL")
 797                 else: #local path
 798                     with open(image_dict['location']) as fimage:
 799                         self.glance.images.upload(new_image.id, fimage)
 800                         #new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 801                         #    container_format="bare", data=fimage, disk_format=disk_format)
 802                 metadata_to_load = image_dict.get('metadata')
 803                 #TODO location is a reserved word for current openstack versions. Use another word
 804                 metadata_to_load['location'] = image_dict['location']
 805                 self.glance.images.update(new_image.id, **metadata_to_load)
 806                 return new_image.id
 807             except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e:
 808                 self._format_exception(e)
 809             except (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 810                 if retry==max_retries:
 811                     continue
 812                 self._format_exception(e)
 813             except IOError as e:  #can not open the file
 814                 raise vimconn.vimconnConnectionException(type(e).__name__ + ": " + str(e)+ " for " + image_dict['location'],
 815                                                          http_code=vimconn.HTTP_Bad_Request)
 816
 817     def delete_image(self, image_id):
 818         '''Deletes a tenant image from openstack VIM. Returns the old id
 819         '''
 820         try:
 821             self._reload_connection()
 822             self.glance.images.delete(image_id)
 823             return image_id
 824         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e: #TODO remove
 825             self._format_exception(e)
 826
 827     def get_image_id_from_path(self, path):
 828         '''Get the image id from image path in the VIM database. Returns the image_id'''
 829         try:
 830             self._reload_connection()
 831             images = self.glance.images.list()
 832             for image in images:
 833                 if image.metadata.get("location")==path:
 834                     return image.id
 835             raise vimconn.vimconnNotFoundException("image with location '{}' not found".format( path))
 836         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 837             self._format_exception(e)
 838
 839     def get_image_list(self, filter_dict={}):
 840         '''Obtain tenant images from VIM
 841         Filter_dict can be:
 842             id: image id
 843             name: image name
 844             checksum: image checksum
 845         Returns the image list of dictionaries:
 846             [{<the fields at Filter_dict plus some VIM specific>}, ...]
 847             List can be empty
 848         '''
 849         self.logger.debug("Getting image list from VIM filter: '%s'", str(filter_dict))
 850         try:
 851             self._reload_connection()
 852             filter_dict_os = filter_dict.copy()
 853             #First we filter by the available filter fields: name, id. The others are removed.
 854             image_list = self.glance.images.list()
 855             filtered_list = []
 856             for image in image_list:
 857                 try:
 858                     if filter_dict.get("name") and image["name"] != filter_dict["name"]:
 859                         continue
 860                     if filter_dict.get("id") and image["id"] != filter_dict["id"]:
 861                         continue
 862                     if filter_dict.get("checksum") and image["checksum"] != filter_dict["checksum"]:
 863                         continue
 864
 865                     filtered_list.append(image.copy())
 866                 except gl1Exceptions.HTTPNotFound:
 867                     pass
 868             return filtered_list
 869         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 870             self._format_exception(e)
 871
 872     def __wait_for_vm(self, vm_id, status):
 873         """wait until vm is in the desired status and return True.
 874         If the VM gets in ERROR status, return false.
 875         If the timeout is reached generate an exception"""
 876         elapsed_time = 0
 877         while elapsed_time < server_timeout:
 878             vm_status = self.nova.servers.get(vm_id).status
 879             if vm_status == status:
 880                 return True
 881             if vm_status == 'ERROR':
 882                 return False
 883             time.sleep(5)
 884             elapsed_time += 5
 885
 886         # if we exceeded the timeout rollback
 887         if elapsed_time >= server_timeout:
 888             raise vimconn.vimconnException('Timeout waiting for instance ' + vm_id + ' to get ' + status,
 889                                            http_code=vimconn.HTTP_Request_Timeout)
 890
 891     def _get_openstack_availablity_zones(self):
 892         """
 893         Get from openstack availability zones available
 894         :return:
 895         """
 896         try:
 897             openstack_availability_zone = self.nova.availability_zones.list()
 898             openstack_availability_zone = [str(zone.zoneName) for zone in openstack_availability_zone
 899                                            if zone.zoneName != 'internal']
 900             return openstack_availability_zone
 901         except Exception as e:
 902             return None
 903
 904     def _set_availablity_zones(self):
 905         """
 906         Set vim availablity zone
 907         :return:
 908         """
 909
 910         if 'availability_zone' in self.config:
 911             vim_availability_zones = self.config.get('availability_zone')
 912             if isinstance(vim_availability_zones, str):
 913                 self.availability_zone = [vim_availability_zones]
 914             elif isinstance(vim_availability_zones, list):
 915                 self.availability_zone = vim_availability_zones
 916         else:
 917             self.availability_zone = self._get_openstack_availablity_zones()
 918
 919     def _get_vm_availability_zone(self, availability_zone_index, availability_zone_list):
 920         """
 921         Return thge availability zone to be used by the created VM.
 922         :return: The VIM availability zone to be used or None
 923         """
 924         if availability_zone_index is None:
 925             if not self.config.get('availability_zone'):
 926                 return None
 927             elif isinstance(self.config.get('availability_zone'), str):
 928                 return self.config['availability_zone']
 929             else:
 930                 # TODO consider using a different parameter at config for default AV and AV list match
 931                 return self.config['availability_zone'][0]
 932
 933         vim_availability_zones = self.availability_zone
 934         # check if VIM offer enough availability zones describe in the VNFD
 935         if vim_availability_zones and len(availability_zone_list) <= len(vim_availability_zones):
 936             # check if all the names of NFV AV match VIM AV names
 937             match_by_index = False
 938             for av in availability_zone_list:
 939                 if av not in vim_availability_zones:
 940                     match_by_index = True
 941                     break
 942             if match_by_index:
 943                 return vim_availability_zones[availability_zone_index]
 944             else:
 945                 return availability_zone_list[availability_zone_index]
 946         else:
 947             raise vimconn.vimconnConflictException("No enough availability zones at VIM for this deployment")
 948
 949     def new_vminstance(self, name, description, start, image_id, flavor_id, net_list, cloud_config=None, disk_list=None,
 950                        availability_zone_index=None, availability_zone_list=None):
 951         """Adds a VM instance to VIM
 952         Params:
 953             start: indicates if VM must start or boot in pause mode. Ignored
 954             image_id,flavor_id: iamge and flavor uuid
 955             net_list: list of interfaces, each one is a dictionary with:
 956                 name:
 957                 net_id: network uuid to connect
 958                 vpci: virtual vcpi to assign, ignored because openstack lack #TODO
 959                 model: interface model, ignored #TODO
 960                 mac_address: used for  SR-IOV ifaces #TODO for other types
 961                 use: 'data', 'bridge',  'mgmt'
 962                 type: 'virtual', 'PCI-PASSTHROUGH'('PF'), 'SR-IOV'('VF'), 'VFnotShared'
 963                 vim_id: filled/added by this function
 964                 floating_ip: True/False (or it can be None)
 965             'cloud_config': (optional) dictionary with:
 966             'key-pairs': (optional) list of strings with the public key to be inserted to the default user
 967             'users': (optional) list of users to be inserted, each item is a dict with:
 968                 'name': (mandatory) user name,
 969                 'key-pairs': (optional) list of strings with the public key to be inserted to the user
 970             'user-data': (optional) string is a text script to be passed directly to cloud-init
 971             'config-files': (optional). List of files to be transferred. Each item is a dict with:
 972                 'dest': (mandatory) string with the destination absolute path
 973                 'encoding': (optional, by default text). Can be one of:
 974                     'b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64'
 975                 'content' (mandatory): string with the content of the file
 976                 'permissions': (optional) string with file permissions, typically octal notation '0644'
 977                 'owner': (optional) file owner, string with the format 'owner:group'
 978             'boot-data-drive': boolean to indicate if user-data must be passed using a boot drive (hard disk)
 979             'disk_list': (optional) list with additional disks to the VM. Each item is a dict with:
 980                 'image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted
 981                 'size': (mandatory) string with the size of the disk in GB
 982                 'vim_id' (optional) should use this existing volume id
 983             availability_zone_index: Index of availability_zone_list to use for this this VM. None if not AV required
 984             availability_zone_list: list of availability zones given by user in the VNFD descriptor.  Ignore if
 985                 availability_zone_index is None
 986                 #TODO ip, security groups
 987         Returns a tuple with the instance identifier and created_items or raises an exception on error
 988             created_items can be None or a dictionary where this method can include key-values that will be passed to
 989             the method delete_vminstance and action_vminstance. Can be used to store created ports, volumes, etc.
 990             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
 991             as not present.
 992         """
 993         self.logger.debug("new_vminstance input: image='%s' flavor='%s' nics='%s'",image_id, flavor_id,str(net_list))
 994         try:
 995             server = None
 996             created_items = {}
 997             # metadata = {}
 998             net_list_vim = []
 999             external_network = []   # list of external networks to be connected to instance, later on used to create floating_ip
1000             no_secured_ports = []   # List of port-is with port-security disabled
1001             self._reload_connection()
1002             # metadata_vpci = {}   # For a specific neutron plugin
1003             block_device_mapping = None
1004             for net in net_list:
1005                 if not net.get("net_id"):   # skip non connected iface
1006                     continue
1007
1008                 port_dict={
1009                     "network_id": net["net_id"],
1010                     "name": net.get("name"),
1011                     "admin_state_up": True
1012                 }
1013                 if net["type"]=="virtual":
1014                     pass
1015                     # if "vpci" in net:
1016                     #     metadata_vpci[ net["net_id"] ] = [[ net["vpci"], "" ]]
1017                 elif net["type"] == "VF" or net["type"] == "SR-IOV":  # for VF
1018                     # if "vpci" in net:
1019                     #     if "VF" not in metadata_vpci:
1020                     #         metadata_vpci["VF"]=[]
1021                     #     metadata_vpci["VF"].append([ net["vpci"], "" ])
1022                     port_dict["binding:vnic_type"]="direct"
1023                     # VIO specific Changes
1024                     if self.vim_type == "VIO":
1025                         # Need to create port with port_security_enabled = False and no-security-groups
1026                         port_dict["port_security_enabled"]=False
1027                         port_dict["provider_security_groups"]=[]
1028                         port_dict["security_groups"]=[]
1029                 else:   # For PT PCI-PASSTHROUGH
1030                     # VIO specific Changes
1031                     # Current VIO release does not support port with type 'direct-physical'
1032                     # So no need to create virtual port in case of PCI-device.
1033                     # Will update port_dict code when support gets added in next VIO release
1034                     if self.vim_type == "VIO":
1035                         raise vimconn.vimconnNotSupportedException(
1036                             "Current VIO release does not support full passthrough (PT)")
1037                     # if "vpci" in net:
1038                     #     if "PF" not in metadata_vpci:
1039                     #         metadata_vpci["PF"]=[]
1040                     #     metadata_vpci["PF"].append([ net["vpci"], "" ])
1041                     port_dict["binding:vnic_type"]="direct-physical"
1042                 if not port_dict["name"]:
1043                     port_dict["name"]=name
1044                 if net.get("mac_address"):
1045                     port_dict["mac_address"]=net["mac_address"]
1046                 if net.get("ip_address"):
1047                     port_dict["fixed_ips"] = [{'ip_address': net["ip_address"]}]
1048                     # TODO add 'subnet_id': <subnet_id>
1049                 new_port = self.neutron.create_port({"port": port_dict })
1050                 created_items["port:" + str(new_port["port"]["id"])] = True
1051                 net["mac_adress"] = new_port["port"]["mac_address"]
1052                 net["vim_id"] = new_port["port"]["id"]
1053                 # if try to use a network without subnetwork, it will return a emtpy list
1054                 fixed_ips = new_port["port"].get("fixed_ips")
1055                 if fixed_ips:
1056                     net["ip"] = fixed_ips[0].get("ip_address")
1057                 else:
1058                     net["ip"] = None
1059
1060                 port = {"port-id": new_port["port"]["id"]}
1061                 if float(self.nova.api_version.get_string()) >= 2.32:
1062                     port["tag"] = new_port["port"]["name"]
1063                 net_list_vim.append(port)
1064
1065                 if net.get('floating_ip', False):
1066                     net['exit_on_floating_ip_error'] = True
1067                     external_network.append(net)
1068                 elif net['use'] == 'mgmt' and self.config.get('use_floating_ip'):
1069                     net['exit_on_floating_ip_error'] = False
1070                     external_network.append(net)
1071                     net['floating_ip'] = self.config.get('use_floating_ip')
1072
1073                 # If port security is disabled when the port has not yet been attached to the VM, then all vm traffic is dropped.
1074                 # As a workaround we wait until the VM is active and then disable the port-security
1075                 if net.get("port_security") == False and not self.config.get("no_port_security_extension"):
1076                     no_secured_ports.append(new_port["port"]["id"])
1077
1078             # if metadata_vpci:
1079             #     metadata = {"pci_assignement": json.dumps(metadata_vpci)}
1080             #     if len(metadata["pci_assignement"]) >255:
1081             #         #limit the metadata size
1082             #         #metadata["pci_assignement"] = metadata["pci_assignement"][0:255]
1083             #         self.logger.warn("Metadata deleted since it exceeds the expected length (255) ")
1084             #         metadata = {}
1085
1086             self.logger.debug("name '%s' image_id '%s'flavor_id '%s' net_list_vim '%s' description '%s'",
1087                               name, image_id, flavor_id, str(net_list_vim), description)
1088
1089             security_groups = self.config.get('security_groups')
1090             if type(security_groups) is str:
1091                 security_groups = ( security_groups, )
1092             # cloud config
1093             config_drive, userdata = self._create_user_data(cloud_config)
1094
1095             # Create additional volumes in case these are present in disk_list
1096             base_disk_index = ord('b')
1097             if disk_list:
1098                 block_device_mapping = {}
1099                 for disk in disk_list:
1100                     if disk.get('vim_id'):
1101                         block_device_mapping['_vd' + chr(base_disk_index)] = disk['vim_id']
1102                     else:
1103                         if 'image_id' in disk:
1104                             volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1105                                                                 chr(base_disk_index), imageRef=disk['image_id'])
1106                         else:
1107                             volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1108                                                                 chr(base_disk_index))
1109                         created_items["volume:" + str(volume.id)] = True
1110                         block_device_mapping['_vd' + chr(base_disk_index)] = volume.id
1111                     base_disk_index += 1
1112
1113                 # Wait until created volumes are with status available
1114                 elapsed_time = 0
1115                 while elapsed_time < volume_timeout:
1116                     for created_item in created_items:
1117                         v, _, volume_id = created_item.partition(":")
1118                         if v == 'volume':
1119                             if self.cinder.volumes.get(volume_id).status != 'available':
1120                                 break
1121                     else:  # all ready: break from while
1122                         break
1123                     time.sleep(5)
1124                     elapsed_time += 5
1125                 # If we exceeded the timeout rollback
1126                 if elapsed_time >= volume_timeout:
1127                     raise vimconn.vimconnException('Timeout creating volumes for instance ' + name,
1128                                                    http_code=vimconn.HTTP_Request_Timeout)
1129             # get availability Zone
1130             vm_av_zone = self._get_vm_availability_zone(availability_zone_index, availability_zone_list)
1131
1132             self.logger.debug("nova.servers.create({}, {}, {}, nics={}, security_groups={}, "
1133                               "availability_zone={}, key_name={}, userdata={}, config_drive={}, "
1134                               "block_device_mapping={})".format(name, image_id, flavor_id, net_list_vim,
1135                                                                 security_groups, vm_av_zone, self.config.get('keypair'),
1136                                                                 userdata, config_drive, block_device_mapping))
1137             server = self.nova.servers.create(name, image_id, flavor_id, nics=net_list_vim,
1138                                               security_groups=security_groups,
1139                                               availability_zone=vm_av_zone,
1140                                               key_name=self.config.get('keypair'),
1141                                               userdata=userdata,
1142                                               config_drive=config_drive,
1143                                               block_device_mapping=block_device_mapping
1144                                               )  # , description=description)
1145
1146             vm_start_time = time.time()
1147             # Previously mentioned workaround to wait until the VM is active and then disable the port-security
1148             if no_secured_ports:
1149                 self.__wait_for_vm(server.id, 'ACTIVE')
1150
1151             for port_id in no_secured_ports:
1152                 try:
1153                     self.neutron.update_port(port_id,
1154                                              {"port": {"port_security_enabled": False, "security_groups": None}})
1155                 except Exception as e:
1156                     raise vimconn.vimconnException("It was not possible to disable port security for port {}".format(
1157                         port_id))
1158             # print "DONE :-)", server
1159
1160             # pool_id = None
1161             if external_network:
1162                 floating_ips = self.neutron.list_floatingips().get("floatingips", ())
1163             for floating_network in external_network:
1164                 try:
1165                     assigned = False
1166                     while not assigned:
1167                         if floating_ips:
1168                             ip = floating_ips.pop(0)
1169                             if ip.get("port_id", False) or ip.get('tenant_id') != server.tenant_id:
1170                                 continue
1171                             if isinstance(floating_network['floating_ip'], str):
1172                                 if ip.get("floating_network_id") != floating_network['floating_ip']:
1173                                     continue
1174                             free_floating_ip = ip.get("floating_ip_address")
1175                         else:
1176                             if isinstance(floating_network['floating_ip'], str) and \
1177                                 floating_network['floating_ip'].lower() != "true":
1178                                 pool_id = floating_network['floating_ip']
1179                             else:
1180                                 # Find the external network
1181                                 external_nets = list()
1182                                 for net in self.neutron.list_networks()['networks']:
1183                                     if net['router:external']:
1184                                             external_nets.append(net)
1185
1186                                 if len(external_nets) == 0:
1187                                     raise vimconn.vimconnException("Cannot create floating_ip automatically since no external "
1188                                                                    "network is present",
1189                                                                     http_code=vimconn.HTTP_Conflict)
1190                                 if len(external_nets) > 1:
1191                                     raise vimconn.vimconnException("Cannot create floating_ip automatically since multiple "
1192                                                                    "external networks are present",
1193                                                                    http_code=vimconn.HTTP_Conflict)
1194
1195                                 pool_id = external_nets[0].get('id')
1196                             param = {'floatingip': {'floating_network_id': pool_id, 'tenant_id': server.tenant_id}}
1197                             try:
1198                                 # self.logger.debug("Creating floating IP")
1199                                 new_floating_ip = self.neutron.create_floatingip(param)
1200                                 free_floating_ip = new_floating_ip['floatingip']['floating_ip_address']
1201                             except Exception as e:
1202                                 raise vimconn.vimconnException(type(e).__name__ + ": Cannot create new floating_ip " +
1203                                                                str(e), http_code=vimconn.HTTP_Conflict)
1204
1205                         fix_ip = floating_network.get('ip')
1206                         while not assigned:
1207                             try:
1208                                 server.add_floating_ip(free_floating_ip, fix_ip)
1209                                 assigned = True
1210                             except Exception as e:
1211                                 # openstack need some time after VM creation to asign an IP. So retry if fails
1212                                 vm_status = self.nova.servers.get(server.id).status
1213                                 if vm_status != 'ACTIVE' and vm_status != 'ERROR':
1214                                     if time.time() - vm_start_time < server_timeout:
1215                                         time.sleep(5)
1216                                         continue
1217                                 raise vimconn.vimconnException(
1218                                     "Cannot create floating_ip: {} {}".format(type(e).__name__, e),
1219                                     http_code=vimconn.HTTP_Conflict)
1220
1221                 except Exception as e:
1222                     if not floating_network['exit_on_floating_ip_error']:
1223                         self.logger.warn("Cannot create floating_ip. %s", str(e))
1224                         continue
1225                     raise
1226
1227             return server.id, created_items
1228 #        except nvExceptions.NotFound as e:
1229 #            error_value=-vimconn.HTTP_Not_Found
1230 #            error_text= "vm instance %s not found" % vm_id
1231 #        except TypeError as e:
1232 #            raise vimconn.vimconnException(type(e).__name__ + ": "+  str(e), http_code=vimconn.HTTP_Bad_Request)
1233
1234         except Exception as e:
1235             server_id = None
1236             if server:
1237                 server_id = server.id
1238             try:
1239                 self.delete_vminstance(server_id, created_items)
1240             except Exception as e2:
1241                 self.logger.error("new_vminstance rollback fail {}".format(e2))
1242
1243             self._format_exception(e)
1244
1245     def get_vminstance(self,vm_id):
1246         '''Returns the VM instance information from VIM'''
1247         #self.logger.debug("Getting VM from VIM")
1248         try:
1249             self._reload_connection()
1250             server = self.nova.servers.find(id=vm_id)
1251             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
1252             return server.to_dict()
1253         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1254             self._format_exception(e)
1255
1256     def get_vminstance_console(self,vm_id, console_type="vnc"):
1257         '''
1258         Get a console for the virtual machine
1259         Params:
1260             vm_id: uuid of the VM
1261             console_type, can be:
1262                 "novnc" (by default), "xvpvnc" for VNC types,
1263                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
1264         Returns dict with the console parameters:
1265                 protocol: ssh, ftp, http, https, ...
1266                 server:   usually ip address
1267                 port:     the http, ssh, ... port
1268                 suffix:   extra text, e.g. the http path and query string
1269         '''
1270         self.logger.debug("Getting VM CONSOLE from VIM")
1271         try:
1272             self._reload_connection()
1273             server = self.nova.servers.find(id=vm_id)
1274             if console_type == None or console_type == "novnc":
1275                 console_dict = server.get_vnc_console("novnc")
1276             elif console_type == "xvpvnc":
1277                 console_dict = server.get_vnc_console(console_type)
1278             elif console_type == "rdp-html5":
1279                 console_dict = server.get_rdp_console(console_type)
1280             elif console_type == "spice-html5":
1281                 console_dict = server.get_spice_console(console_type)
1282             else:
1283                 raise vimconn.vimconnException("console type '{}' not allowed".format(console_type), http_code=vimconn.HTTP_Bad_Request)
1284
1285             console_dict1 = console_dict.get("console")
1286             if console_dict1:
1287                 console_url = console_dict1.get("url")
1288                 if console_url:
1289                     #parse console_url
1290                     protocol_index = console_url.find("//")
1291                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1292                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1293                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1294                         return -vimconn.HTTP_Internal_Server_Error, "Unexpected response from VIM"
1295                     console_dict={"protocol": console_url[0:protocol_index],
1296                                   "server":   console_url[protocol_index+2:port_index],
1297                                   "port":     console_url[port_index:suffix_index],
1298                                   "suffix":   console_url[suffix_index+1:]
1299                                   }
1300                     protocol_index += 2
1301                     return console_dict
1302             raise vimconn.vimconnUnexpectedResponse("Unexpected response from VIM")
1303
1304         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.BadRequest, ConnectionError) as e:
1305             self._format_exception(e)
1306
1307     def delete_vminstance(self, vm_id, created_items=None):
1308         '''Removes a VM instance from VIM. Returns the old identifier
1309         '''
1310         #print "osconnector: Getting VM from VIM"
1311         if created_items == None:
1312             created_items = {}
1313         try:
1314             self._reload_connection()
1315             # delete VM ports attached to this networks before the virtual machine
1316             for k, v in created_items.items():
1317                 if not v:  # skip already deleted
1318                     continue
1319                 try:
1320                     k_item, _, k_id = k.partition(":")
1321                     if k_item == "port":
1322                         self.neutron.delete_port(k_id)
1323                 except Exception as e:
1324                     self.logger.error("Error deleting port: {}: {}".format(type(e).__name__, e))
1325
1326             # #commented because detaching the volumes makes the servers.delete not work properly ?!?
1327             # #dettach volumes attached
1328             # server = self.nova.servers.get(vm_id)
1329             # volumes_attached_dict = server._info['os-extended-volumes:volumes_attached']   #volume['id']
1330             # #for volume in volumes_attached_dict:
1331             # #    self.cinder.volumes.detach(volume['id'])
1332
1333             if vm_id:
1334                 self.nova.servers.delete(vm_id)
1335
1336             # delete volumes. Although having detached, they should have in active status before deleting
1337             # we ensure in this loop
1338             keep_waiting = True
1339             elapsed_time = 0
1340             while keep_waiting and elapsed_time < volume_timeout:
1341                 keep_waiting = False
1342                 for k, v in created_items.items():
1343                     if not v:  # skip already deleted
1344                         continue
1345                     try:
1346                         k_item, _, k_id = k.partition(":")
1347                         if k_item == "volume":
1348                             if self.cinder.volumes.get(k_id).status != 'available':
1349                                 keep_waiting = True
1350                             else:
1351                                 self.cinder.volumes.delete(k_id)
1352                     except Exception as e:
1353                         self.logger.error("Error deleting volume: {}: {}".format(type(e).__name__, e))
1354                 if keep_waiting:
1355                     time.sleep(1)
1356                     elapsed_time += 1
1357             return None
1358         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
1359             self._format_exception(e)
1360
1361     def refresh_vms_status(self, vm_list):
1362         '''Get the status of the virtual machines and their interfaces/ports
1363            Params: the list of VM identifiers
1364            Returns a dictionary with:
1365                 vm_id:          #VIM id of this Virtual Machine
1366                     status:     #Mandatory. Text with one of:
1367                                 #  DELETED (not found at vim)
1368                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1369                                 #  OTHER (Vim reported other status not understood)
1370                                 #  ERROR (VIM indicates an ERROR status)
1371                                 #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
1372                                 #  CREATING (on building process), ERROR
1373                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
1374                                 #
1375                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
1376                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
1377                     interfaces:
1378                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
1379                         mac_address:      #Text format XX:XX:XX:XX:XX:XX
1380                         vim_net_id:       #network id where this interface is connected
1381                         vim_interface_id: #interface/port VIM id
1382                         ip_address:       #null, or text with IPv4, IPv6 address
1383                         compute_node:     #identification of compute node where PF,VF interface is allocated
1384                         pci:              #PCI address of the NIC that hosts the PF,VF
1385                         vlan:             #physical VLAN used for VF
1386         '''
1387         vm_dict={}
1388         self.logger.debug("refresh_vms status: Getting tenant VM instance information from VIM")
1389         for vm_id in vm_list:
1390             vm={}
1391             try:
1392                 vm_vim = self.get_vminstance(vm_id)
1393                 if vm_vim['status'] in vmStatus2manoFormat:
1394                     vm['status']    =  vmStatus2manoFormat[ vm_vim['status'] ]
1395                 else:
1396                     vm['status']    = "OTHER"
1397                     vm['error_msg'] = "VIM status reported " + vm_vim['status']
1398                 try:
1399                     vm['vim_info']  = yaml.safe_dump(vm_vim, default_flow_style=True, width=256)
1400                 except yaml.representer.RepresenterError:
1401                     vm['vim_info'] = str(vm_vim)
1402                 vm["interfaces"] = []
1403                 if vm_vim.get('fault'):
1404                     vm['error_msg'] = str(vm_vim['fault'])
1405                 #get interfaces
1406                 try:
1407                     self._reload_connection()
1408                     port_dict = self.neutron.list_ports(device_id=vm_id)
1409                     for port in port_dict["ports"]:
1410                         interface={}
1411                         try:
1412                             interface['vim_info'] = yaml.safe_dump(port, default_flow_style=True, width=256)
1413                         except yaml.representer.RepresenterError:
1414                             interface['vim_info'] = str(port)
1415                         interface["mac_address"] = port.get("mac_address")
1416                         interface["vim_net_id"] = port["network_id"]
1417                         interface["vim_interface_id"] = port["id"]
1418                         # check if OS-EXT-SRV-ATTR:host is there,
1419                         # in case of non-admin credentials, it will be missing
1420                         if vm_vim.get('OS-EXT-SRV-ATTR:host'):
1421                             interface["compute_node"] = vm_vim['OS-EXT-SRV-ATTR:host']
1422                         interface["pci"] = None
1423
1424                         # check if binding:profile is there,
1425                         # in case of non-admin credentials, it will be missing
1426                         if port.get('binding:profile'):
1427                             if port['binding:profile'].get('pci_slot'):
1428                                 # TODO: At the moment sr-iov pci addresses are converted to PF pci addresses by setting the slot to 0x00
1429                                 # TODO: This is just a workaround valid for niantinc. Find a better way to do so
1430                                 #   CHANGE DDDD:BB:SS.F to DDDD:BB:00.(F%2)   assuming there are 2 ports per nic
1431                                 pci = port['binding:profile']['pci_slot']
1432                                 # interface["pci"] = pci[:-4] + "00." + str(int(pci[-1]) % 2)
1433                                 interface["pci"] = pci
1434                         interface["vlan"] = None
1435                         #if network is of type vlan and port is of type direct (sr-iov) then set vlan id
1436                         network = self.neutron.show_network(port["network_id"])
1437                         if network['network'].get('provider:network_type') == 'vlan' and \
1438                             port.get("binding:vnic_type") == "direct":
1439                             interface["vlan"] = network['network'].get('provider:segmentation_id')
1440                         ips=[]
1441                         #look for floating ip address
1442                         try:
1443                             floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
1444                             if floating_ip_dict.get("floatingips"):
1445                                 ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address") )
1446                         except Exception:
1447                             pass
1448
1449                         for subnet in port["fixed_ips"]:
1450                             ips.append(subnet["ip_address"])
1451                         interface["ip_address"] = ";".join(ips)
1452                         vm["interfaces"].append(interface)
1453                 except Exception as e:
1454                     self.logger.error("Error getting vm interface information {}: {}".format(type(e).__name__, e),
1455                                       exc_info=True)
1456             except vimconn.vimconnNotFoundException as e:
1457                 self.logger.error("Exception getting vm status: %s", str(e))
1458                 vm['status'] = "DELETED"
1459                 vm['error_msg'] = str(e)
1460             except vimconn.vimconnException as e:
1461                 self.logger.error("Exception getting vm status: %s", str(e))
1462                 vm['status'] = "VIM_ERROR"
1463                 vm['error_msg'] = str(e)
1464             vm_dict[vm_id] = vm
1465         return vm_dict
1466
1467     def action_vminstance(self, vm_id, action_dict, created_items={}):
1468         '''Send and action over a VM instance from VIM
1469         Returns None or the console dict if the action was successfully sent to the VIM'''
1470         self.logger.debug("Action over VM '%s': %s", vm_id, str(action_dict))
1471         try:
1472             self._reload_connection()
1473             server = self.nova.servers.find(id=vm_id)
1474             if "start" in action_dict:
1475                 if action_dict["start"]=="rebuild":
1476                     server.rebuild()
1477                 else:
1478                     if server.status=="PAUSED":
1479                         server.unpause()
1480                     elif server.status=="SUSPENDED":
1481                         server.resume()
1482                     elif server.status=="SHUTOFF":
1483                         server.start()
1484             elif "pause" in action_dict:
1485                 server.pause()
1486             elif "resume" in action_dict:
1487                 server.resume()
1488             elif "shutoff" in action_dict or "shutdown" in action_dict:
1489                 server.stop()
1490             elif "forceOff" in action_dict:
1491                 server.stop() #TODO
1492             elif "terminate" in action_dict:
1493                 server.delete()
1494             elif "createImage" in action_dict:
1495                 server.create_image()
1496                 #"path":path_schema,
1497                 #"description":description_schema,
1498                 #"name":name_schema,
1499                 #"metadata":metadata_schema,
1500                 #"imageRef": id_schema,
1501                 #"disk": {"oneOf":[{"type": "null"}, {"type":"string"}] },
1502             elif "rebuild" in action_dict:
1503                 server.rebuild(server.image['id'])
1504             elif "reboot" in action_dict:
1505                 server.reboot() #reboot_type='SOFT'
1506             elif "console" in action_dict:
1507                 console_type = action_dict["console"]
1508                 if console_type == None or console_type == "novnc":
1509                     console_dict = server.get_vnc_console("novnc")
1510                 elif console_type == "xvpvnc":
1511                     console_dict = server.get_vnc_console(console_type)
1512                 elif console_type == "rdp-html5":
1513                     console_dict = server.get_rdp_console(console_type)
1514                 elif console_type == "spice-html5":
1515                     console_dict = server.get_spice_console(console_type)
1516                 else:
1517                     raise vimconn.vimconnException("console type '{}' not allowed".format(console_type),
1518                                                    http_code=vimconn.HTTP_Bad_Request)
1519                 try:
1520                     console_url = console_dict["console"]["url"]
1521                     #parse console_url
1522                     protocol_index = console_url.find("//")
1523                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1524                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1525                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1526                         raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1527                     console_dict2={"protocol": console_url[0:protocol_index],
1528                                   "server":   console_url[protocol_index+2 : port_index],
1529                                   "port":     int(console_url[port_index+1 : suffix_index]),
1530                                   "suffix":   console_url[suffix_index+1:]
1531                                   }
1532                     return console_dict2
1533                 except Exception as e:
1534                     raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1535
1536             return None
1537         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1538             self._format_exception(e)
1539         #TODO insert exception vimconn.HTTP_Unauthorized
1540
1541     ####### VIO Specific Changes #########
1542     def _genrate_vlanID(self):
1543         """
1544          Method to get unused vlanID
1545             Args:
1546                 None
1547             Returns:
1548                 vlanID
1549         """
1550         #Get used VLAN IDs
1551         usedVlanIDs = []
1552         networks = self.get_network_list()
1553         for net in networks:
1554             if net.get('provider:segmentation_id'):
1555                 usedVlanIDs.append(net.get('provider:segmentation_id'))
1556         used_vlanIDs = set(usedVlanIDs)
1557
1558         #find unused VLAN ID
1559         for vlanID_range in self.config.get('dataplane_net_vlan_range'):
1560             try:
1561                 start_vlanid , end_vlanid = map(int, vlanID_range.replace(" ", "").split("-"))
1562                 for vlanID in xrange(start_vlanid, end_vlanid + 1):
1563                     if vlanID not in used_vlanIDs:
1564                         return vlanID
1565             except Exception as exp:
1566                 raise vimconn.vimconnException("Exception {} occurred while generating VLAN ID.".format(exp))
1567         else:
1568             raise vimconn.vimconnConflictException("Unable to create the SRIOV VLAN network."\
1569                 " All given Vlan IDs {} are in use.".format(self.config.get('dataplane_net_vlan_range')))
1570
1571
1572     def _validate_vlan_ranges(self, dataplane_net_vlan_range):
1573         """
1574         Method to validate user given vlanID ranges
1575             Args:  None
1576             Returns: None
1577         """
1578         for vlanID_range in dataplane_net_vlan_range:
1579             vlan_range = vlanID_range.replace(" ", "")
1580             #validate format
1581             vlanID_pattern = r'(\d)*-(\d)*$'
1582             match_obj = re.match(vlanID_pattern, vlan_range)
1583             if not match_obj:
1584                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}.You must provide "\
1585                 "'dataplane_net_vlan_range' in format [start_ID - end_ID].".format(vlanID_range))
1586
1587             start_vlanid , end_vlanid = map(int,vlan_range.split("-"))
1588             if start_vlanid <= 0 :
1589                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1590                 "Start ID can not be zero. For VLAN "\
1591                 "networks valid IDs are 1 to 4094 ".format(vlanID_range))
1592             if end_vlanid > 4094 :
1593                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1594                 "End VLAN ID can not be greater than 4094. For VLAN "\
1595                 "networks valid IDs are 1 to 4094 ".format(vlanID_range))
1596
1597             if start_vlanid > end_vlanid:
1598                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1599                     "You must provide a 'dataplane_net_vlan_range' in format start_ID - end_ID and "\
1600                     "start_ID < end_ID ".format(vlanID_range))
1601
1602 #NOT USED FUNCTIONS
1603
1604     def new_external_port(self, port_data):
1605         #TODO openstack if needed
1606         '''Adds a external port to VIM'''
1607         '''Returns the port identifier'''
1608         return -vimconn.HTTP_Internal_Server_Error, "osconnector.new_external_port() not implemented"
1609
1610     def connect_port_network(self, port_id, network_id, admin=False):
1611         #TODO openstack if needed
1612         '''Connects a external port to a network'''
1613         '''Returns status code of the VIM response'''
1614         return -vimconn.HTTP_Internal_Server_Error, "osconnector.connect_port_network() not implemented"
1615
1616     def new_user(self, user_name, user_passwd, tenant_id=None):
1617         '''Adds a new user to openstack VIM'''
1618         '''Returns the user identifier'''
1619         self.logger.debug("osconnector: Adding a new user to VIM")
1620         try:
1621             self._reload_connection()
1622             user=self.keystone.users.create(user_name, password=user_passwd, default_project=tenant_id)
1623             #self.keystone.tenants.add_user(self.k_creds["username"], #role)
1624             return user.id
1625         except ksExceptions.ConnectionError as e:
1626             error_value=-vimconn.HTTP_Bad_Request
1627             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1628         except ksExceptions.ClientException as e: #TODO remove
1629             error_value=-vimconn.HTTP_Bad_Request
1630             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1631         #TODO insert exception vimconn.HTTP_Unauthorized
1632         #if reaching here is because an exception
1633         self.logger.debug("new_user " + error_text)
1634         return error_value, error_text
1635
1636     def delete_user(self, user_id):
1637         '''Delete a user from openstack VIM'''
1638         '''Returns the user identifier'''
1639         if self.debug:
1640             print("osconnector: Deleting  a  user from VIM")
1641         try:
1642             self._reload_connection()
1643             self.keystone.users.delete(user_id)
1644             return 1, user_id
1645         except ksExceptions.ConnectionError as e:
1646             error_value=-vimconn.HTTP_Bad_Request
1647             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1648         except ksExceptions.NotFound as e:
1649             error_value=-vimconn.HTTP_Not_Found
1650             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1651         except ksExceptions.ClientException as e: #TODO remove
1652             error_value=-vimconn.HTTP_Bad_Request
1653             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1654         #TODO insert exception vimconn.HTTP_Unauthorized
1655         #if reaching here is because an exception
1656             self.logger.debug("delete_tenant " + error_text)
1657         return error_value, error_text
1658
1659     def get_hosts_info(self):
1660         '''Get the information of deployed hosts
1661         Returns the hosts content'''
1662         if self.debug:
1663             print("osconnector: Getting Host info from VIM")
1664         try:
1665             h_list=[]
1666             self._reload_connection()
1667             hypervisors = self.nova.hypervisors.list()
1668             for hype in hypervisors:
1669                 h_list.append( hype.to_dict() )
1670             return 1, {"hosts":h_list}
1671         except nvExceptions.NotFound as e:
1672             error_value=-vimconn.HTTP_Not_Found
1673             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1674         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1675             error_value=-vimconn.HTTP_Bad_Request
1676             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1677         #TODO insert exception vimconn.HTTP_Unauthorized
1678         #if reaching here is because an exception
1679         self.logger.debug("get_hosts_info " + error_text)
1680         return error_value, error_text
1681
1682     def get_hosts(self, vim_tenant):
1683         '''Get the hosts and deployed instances
1684         Returns the hosts content'''
1685         r, hype_dict = self.get_hosts_info()
1686         if r<0:
1687             return r, hype_dict
1688         hypervisors = hype_dict["hosts"]
1689         try:
1690             servers = self.nova.servers.list()
1691             for hype in hypervisors:
1692                 for server in servers:
1693                     if server.to_dict()['OS-EXT-SRV-ATTR:hypervisor_hostname']==hype['hypervisor_hostname']:
1694                         if 'vm' in hype:
1695                             hype['vm'].append(server.id)
1696                         else:
1697                             hype['vm'] = [server.id]
1698             return 1, hype_dict
1699         except nvExceptions.NotFound as e:
1700             error_value=-vimconn.HTTP_Not_Found
1701             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1702         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1703             error_value=-vimconn.HTTP_Bad_Request
1704             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1705         #TODO insert exception vimconn.HTTP_Unauthorized
1706         #if reaching here is because an exception
1707         self.logger.debug("get_hosts " + error_text)
1708         return error_value, error_text
1709
1710     def new_classification(self, name, ctype, definition):
1711         self.logger.debug(
1712             'Adding a new (Traffic) Classification to VIM, named %s', name)
1713         try:
1714             new_class = None
1715             self._reload_connection()
1716             if ctype not in supportedClassificationTypes:
1717                 raise vimconn.vimconnNotSupportedException(
1718                         'OpenStack VIM connector doesn\'t support provided '
1719                         'Classification Type {}, supported ones are: '
1720                         '{}'.format(ctype, supportedClassificationTypes))
1721             if not self._validate_classification(ctype, definition):
1722                 raise vimconn.vimconnException(
1723                     'Incorrect Classification definition '
1724                     'for the type specified.')
1725             classification_dict = definition
1726             classification_dict['name'] = name
1727
1728             new_class = self.neutron.create_sfc_flow_classifier(
1729                 {'flow_classifier': classification_dict})
1730             return new_class['flow_classifier']['id']
1731         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1732                 neExceptions.NeutronException, ConnectionError) as e:
1733             self.logger.error(
1734                 'Creation of Classification failed.')
1735             self._format_exception(e)
1736
1737     def get_classification(self, class_id):
1738         self.logger.debug(" Getting Classification %s from VIM", class_id)
1739         filter_dict = {"id": class_id}
1740         class_list = self.get_classification_list(filter_dict)
1741         if len(class_list) == 0:
1742             raise vimconn.vimconnNotFoundException(
1743                 "Classification '{}' not found".format(class_id))
1744         elif len(class_list) > 1:
1745             raise vimconn.vimconnConflictException(
1746                 "Found more than one Classification with this criteria")
1747         classification = class_list[0]
1748         return classification
1749
1750     def get_classification_list(self, filter_dict={}):
1751         self.logger.debug("Getting Classifications from VIM filter: '%s'",
1752                           str(filter_dict))
1753         try:
1754             filter_dict_os = filter_dict.copy()
1755             self._reload_connection()
1756             if self.api_version3 and "tenant_id" in filter_dict_os:
1757                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1758             classification_dict = self.neutron.list_sfc_flow_classifiers(
1759                 **filter_dict_os)
1760             classification_list = classification_dict["flow_classifiers"]
1761             self.__classification_os2mano(classification_list)
1762             return classification_list
1763         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1764                 neExceptions.NeutronException, ConnectionError) as e:
1765             self._format_exception(e)
1766
1767     def delete_classification(self, class_id):
1768         self.logger.debug("Deleting Classification '%s' from VIM", class_id)
1769         try:
1770             self._reload_connection()
1771             self.neutron.delete_sfc_flow_classifier(class_id)
1772             return class_id
1773         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1774                 ksExceptions.ClientException, neExceptions.NeutronException,
1775                 ConnectionError) as e:
1776             self._format_exception(e)
1777
1778     def new_sfi(self, name, ingress_ports, egress_ports, sfc_encap=True):
1779         self.logger.debug(
1780             "Adding a new Service Function Instance to VIM, named '%s'", name)
1781         try:
1782             new_sfi = None
1783             self._reload_connection()
1784             correlation = None
1785             if sfc_encap:
1786                 correlation = 'nsh'
1787             if len(ingress_ports) != 1:
1788                 raise vimconn.vimconnNotSupportedException(
1789                     "OpenStack VIM connector can only have "
1790                     "1 ingress port per SFI")
1791             if len(egress_ports) != 1:
1792                 raise vimconn.vimconnNotSupportedException(
1793                     "OpenStack VIM connector can only have "
1794                     "1 egress port per SFI")
1795             sfi_dict = {'name': name,
1796                         'ingress': ingress_ports[0],
1797                         'egress': egress_ports[0],
1798                         'service_function_parameters': {
1799                             'correlation': correlation}}
1800             new_sfi = self.neutron.create_sfc_port_pair({'port_pair': sfi_dict})
1801             return new_sfi['port_pair']['id']
1802         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1803                 neExceptions.NeutronException, ConnectionError) as e:
1804             if new_sfi:
1805                 try:
1806                     self.neutron.delete_sfc_port_pair(
1807                         new_sfi['port_pair']['id'])
1808                 except Exception:
1809                     self.logger.error(
1810                         'Creation of Service Function Instance failed, with '
1811                         'subsequent deletion failure as well.')
1812             self._format_exception(e)
1813
1814     def get_sfi(self, sfi_id):
1815         self.logger.debug(
1816             'Getting Service Function Instance %s from VIM', sfi_id)
1817         filter_dict = {"id": sfi_id}
1818         sfi_list = self.get_sfi_list(filter_dict)
1819         if len(sfi_list) == 0:
1820             raise vimconn.vimconnNotFoundException(
1821                 "Service Function Instance '{}' not found".format(sfi_id))
1822         elif len(sfi_list) > 1:
1823             raise vimconn.vimconnConflictException(
1824                 'Found more than one Service Function Instance '
1825                 'with this criteria')
1826         sfi = sfi_list[0]
1827         return sfi
1828
1829     def get_sfi_list(self, filter_dict={}):
1830         self.logger.debug("Getting Service Function Instances from "
1831                           "VIM filter: '%s'", str(filter_dict))
1832         try:
1833             self._reload_connection()
1834             filter_dict_os = filter_dict.copy()
1835             if self.api_version3 and "tenant_id" in filter_dict_os:
1836                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1837             sfi_dict = self.neutron.list_sfc_port_pairs(**filter_dict_os)
1838             sfi_list = sfi_dict["port_pairs"]
1839             self.__sfi_os2mano(sfi_list)
1840             return sfi_list
1841         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1842                 neExceptions.NeutronException, ConnectionError) as e:
1843             self._format_exception(e)
1844
1845     def delete_sfi(self, sfi_id):
1846         self.logger.debug("Deleting Service Function Instance '%s' "
1847                           "from VIM", sfi_id)
1848         try:
1849             self._reload_connection()
1850             self.neutron.delete_sfc_port_pair(sfi_id)
1851             return sfi_id
1852         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1853                 ksExceptions.ClientException, neExceptions.NeutronException,
1854                 ConnectionError) as e:
1855             self._format_exception(e)
1856
1857     def new_sf(self, name, sfis, sfc_encap=True):
1858         self.logger.debug("Adding a new Service Function to VIM, "
1859                           "named '%s'", name)
1860         try:
1861             new_sf = None
1862             self._reload_connection()
1863             # correlation = None
1864             # if sfc_encap:
1865             #     correlation = 'nsh'
1866             for instance in sfis:
1867                 sfi = self.get_sfi(instance)
1868                 if sfi.get('sfc_encap') != sfc_encap:
1869                     raise vimconn.vimconnNotSupportedException(
1870                         "OpenStack VIM connector requires all SFIs of the "
1871                         "same SF to share the same SFC Encapsulation")
1872             sf_dict = {'name': name,
1873                        'port_pairs': sfis}
1874             new_sf = self.neutron.create_sfc_port_pair_group({
1875                 'port_pair_group': sf_dict})
1876             return new_sf['port_pair_group']['id']
1877         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1878                 neExceptions.NeutronException, ConnectionError) as e:
1879             if new_sf:
1880                 try:
1881                     self.neutron.delete_sfc_port_pair_group(
1882                         new_sf['port_pair_group']['id'])
1883                 except Exception:
1884                     self.logger.error(
1885                         'Creation of Service Function failed, with '
1886                         'subsequent deletion failure as well.')
1887             self._format_exception(e)
1888
1889     def get_sf(self, sf_id):
1890         self.logger.debug("Getting Service Function %s from VIM", sf_id)
1891         filter_dict = {"id": sf_id}
1892         sf_list = self.get_sf_list(filter_dict)
1893         if len(sf_list) == 0:
1894             raise vimconn.vimconnNotFoundException(
1895                 "Service Function '{}' not found".format(sf_id))
1896         elif len(sf_list) > 1:
1897             raise vimconn.vimconnConflictException(
1898                 "Found more than one Service Function with this criteria")
1899         sf = sf_list[0]
1900         return sf
1901
1902     def get_sf_list(self, filter_dict={}):
1903         self.logger.debug("Getting Service Function from VIM filter: '%s'",
1904                           str(filter_dict))
1905         try:
1906             self._reload_connection()
1907             filter_dict_os = filter_dict.copy()
1908             if self.api_version3 and "tenant_id" in filter_dict_os:
1909                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1910             sf_dict = self.neutron.list_sfc_port_pair_groups(**filter_dict_os)
1911             sf_list = sf_dict["port_pair_groups"]
1912             self.__sf_os2mano(sf_list)
1913             return sf_list
1914         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1915                 neExceptions.NeutronException, ConnectionError) as e:
1916             self._format_exception(e)
1917
1918     def delete_sf(self, sf_id):
1919         self.logger.debug("Deleting Service Function '%s' from VIM", sf_id)
1920         try:
1921             self._reload_connection()
1922             self.neutron.delete_sfc_port_pair_group(sf_id)
1923             return sf_id
1924         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1925                 ksExceptions.ClientException, neExceptions.NeutronException,
1926                 ConnectionError) as e:
1927             self._format_exception(e)
1928
1929     def new_sfp(self, name, classifications, sfs, sfc_encap=True, spi=None):
1930         self.logger.debug("Adding a new Service Function Path to VIM, "
1931                           "named '%s'", name)
1932         try:
1933             new_sfp = None
1934             self._reload_connection()
1935             # In networking-sfc the MPLS encapsulation is legacy
1936             # should be used when no full SFC Encapsulation is intended
1937             sfc_encap = 'mpls'
1938             if sfc_encap:
1939                 correlation = 'nsh'
1940             sfp_dict = {'name': name,
1941                         'flow_classifiers': classifications,
1942                         'port_pair_groups': sfs,
1943                         'chain_parameters': {'correlation': correlation}}
1944             if spi:
1945                 sfp_dict['chain_id'] = spi
1946             new_sfp = self.neutron.create_sfc_port_chain({'port_chain': sfp_dict})
1947             return new_sfp["port_chain"]["id"]
1948         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1949                 neExceptions.NeutronException, ConnectionError) as e:
1950             if new_sfp:
1951                 try:
1952                     self.neutron.delete_sfc_port_chain(new_sfp['port_chain']['id'])
1953                 except Exception:
1954                     self.logger.error(
1955                         'Creation of Service Function Path failed, with '
1956                         'subsequent deletion failure as well.')
1957             self._format_exception(e)
1958
1959     def get_sfp(self, sfp_id):
1960         self.logger.debug(" Getting Service Function Path %s from VIM", sfp_id)
1961         filter_dict = {"id": sfp_id}
1962         sfp_list = self.get_sfp_list(filter_dict)
1963         if len(sfp_list) == 0:
1964             raise vimconn.vimconnNotFoundException(
1965                 "Service Function Path '{}' not found".format(sfp_id))
1966         elif len(sfp_list) > 1:
1967             raise vimconn.vimconnConflictException(
1968                 "Found more than one Service Function Path with this criteria")
1969         sfp = sfp_list[0]
1970         return sfp
1971
1972     def get_sfp_list(self, filter_dict={}):
1973         self.logger.debug("Getting Service Function Paths from VIM filter: "
1974                           "'%s'", str(filter_dict))
1975         try:
1976             self._reload_connection()
1977             filter_dict_os = filter_dict.copy()
1978             if self.api_version3 and "tenant_id" in filter_dict_os:
1979                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1980             sfp_dict = self.neutron.list_sfc_port_chains(**filter_dict_os)
1981             sfp_list = sfp_dict["port_chains"]
1982             self.__sfp_os2mano(sfp_list)
1983             return sfp_list
1984         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1985                 neExceptions.NeutronException, ConnectionError) as e:
1986             self._format_exception(e)
1987
1988     def delete_sfp(self, sfp_id):
1989         self.logger.debug(
1990             "Deleting Service Function Path '%s' from VIM", sfp_id)
1991         try:
1992             self._reload_connection()
1993             self.neutron.delete_sfc_port_chain(sfp_id)
1994             return sfp_id
1995         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1996                 ksExceptions.ClientException, neExceptions.NeutronException,
1997                 ConnectionError) as e:
1998             self._format_exception(e)