osm_ro/vimconn_openstack.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 osconnector implements all the methods to interact with openstack using the python-neutronclient.
  26
  27 For the VNF forwarding graph, The OpenStack VIM connector calls the
  28 networking-sfc Neutron extension methods, whose resources are mapped
  29 to the VIM connector's SFC resources as follows:
  30 - Classification (OSM) -> Flow Classifier (Neutron)
  31 - Service Function Instance (OSM) -> Port Pair (Neutron)
  32 - Service Function (OSM) -> Port Pair Group (Neutron)
  33 - Service Function Path (OSM) -> Port Chain (Neutron)
  34 '''
  35 __author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes, xFlow Research, Igor D.C."
  36 __date__  = "$22-sep-2017 23:59:59$"
  37
  38 import vimconn
  39 import json
  40 import logging
  41 import netaddr
  42 import time
  43 import yaml
  44 import random
  45 import sys
  46 import re
  47 import copy
  48
  49 from novaclient import client as nClient, exceptions as nvExceptions
  50 from keystoneauth1.identity import v2, v3
  51 from keystoneauth1 import session
  52 import keystoneclient.exceptions as ksExceptions
  53 import keystoneclient.v3.client as ksClient_v3
  54 import keystoneclient.v2_0.client as ksClient_v2
  55 from glanceclient import client as glClient
  56 import glanceclient.client as gl1Client
  57 import glanceclient.exc as gl1Exceptions
  58 from  cinderclient import client as cClient
  59 from httplib import HTTPException
  60 from neutronclient.neutron import client as neClient
  61 from neutronclient.common import exceptions as neExceptions
  62 from requests.exceptions import ConnectionError
  63 from email.mime.multipart import MIMEMultipart
  64 from email.mime.text import MIMEText
  65
  66
  67 """contain the openstack virtual machine status to openmano status"""
  68 vmStatus2manoFormat={'ACTIVE':'ACTIVE',
  69                      'PAUSED':'PAUSED',
  70                      'SUSPENDED': 'SUSPENDED',
  71                      'SHUTOFF':'INACTIVE',
  72                      'BUILD':'BUILD',
  73                      'ERROR':'ERROR','DELETED':'DELETED'
  74                      }
  75 netStatus2manoFormat={'ACTIVE':'ACTIVE','PAUSED':'PAUSED','INACTIVE':'INACTIVE','BUILD':'BUILD','ERROR':'ERROR','DELETED':'DELETED'
  76                      }
  77
  78 supportedClassificationTypes = ['legacy_flow_classifier']
  79
  80 #global var to have a timeout creating and deleting volumes
  81 volume_timeout = 60
  82 server_timeout = 300
  83
  84 class vimconnector(vimconn.vimconnector):
  85     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None,
  86                  log_level=None, config={}, persistent_info={}):
  87         '''using common constructor parameters. In this case
  88         'url' is the keystone authorization url,
  89         'url_admin' is not use
  90         '''
  91         api_version = config.get('APIversion')
  92         if api_version and api_version not in ('v3.3', 'v2.0', '2', '3'):
  93             raise vimconn.vimconnException("Invalid value '{}' for config:APIversion. "
  94                                            "Allowed values are 'v3.3', 'v2.0', '2' or '3'".format(api_version))
  95         vim_type = config.get('vim_type')
  96         if vim_type and vim_type not in ('vio', 'VIO'):
  97             raise vimconn.vimconnException("Invalid value '{}' for config:vim_type."
  98                             "Allowed values are 'vio' or 'VIO'".format(vim_type))
  99
 100         if config.get('dataplane_net_vlan_range') is not None:
 101             #validate vlan ranges provided by user
 102             self._validate_vlan_ranges(config.get('dataplane_net_vlan_range'))
 103
 104         vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level,
 105                                       config)
 106
 107         self.insecure = self.config.get("insecure", False)
 108         if not url:
 109             raise TypeError('url param can not be NoneType')
 110         self.persistent_info = persistent_info
 111         self.availability_zone = persistent_info.get('availability_zone', None)
 112         self.session = persistent_info.get('session', {'reload_client': True})
 113         self.nova = self.session.get('nova')
 114         self.neutron = self.session.get('neutron')
 115         self.cinder = self.session.get('cinder')
 116         self.glance = self.session.get('glance')
 117         self.glancev1 = self.session.get('glancev1')
 118         self.keystone = self.session.get('keystone')
 119         self.api_version3 = self.session.get('api_version3')
 120         self.vim_type = self.config.get("vim_type")
 121         if self.vim_type:
 122             self.vim_type = self.vim_type.upper()
 123         if self.config.get("use_internal_endpoint"):
 124             self.endpoint_type = "internalURL"
 125         else:
 126             self.endpoint_type = None
 127
 128         self.logger = logging.getLogger('openmano.vim.openstack')
 129
 130         ####### VIO Specific Changes #########
 131         if self.vim_type == "VIO":
 132             self.logger = logging.getLogger('openmano.vim.vio')
 133
 134         if log_level:
 135             self.logger.setLevel( getattr(logging, log_level))
 136
 137     def __getitem__(self, index):
 138         """Get individuals parameters.
 139         Throw KeyError"""
 140         if index == 'project_domain_id':
 141             return self.config.get("project_domain_id")
 142         elif index == 'user_domain_id':
 143             return self.config.get("user_domain_id")
 144         else:
 145             return vimconn.vimconnector.__getitem__(self, index)
 146
 147     def __setitem__(self, index, value):
 148         """Set individuals parameters and it is marked as dirty so to force connection reload.
 149         Throw KeyError"""
 150         if index == 'project_domain_id':
 151             self.config["project_domain_id"] = value
 152         elif index == 'user_domain_id':
 153                 self.config["user_domain_id"] = value
 154         else:
 155             vimconn.vimconnector.__setitem__(self, index, value)
 156         self.session['reload_client'] = True
 157
 158     def _reload_connection(self):
 159         '''Called before any operation, it check if credentials has changed
 160         Throw keystoneclient.apiclient.exceptions.AuthorizationFailure
 161         '''
 162         #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-)
 163         if self.session['reload_client']:
 164             if self.config.get('APIversion'):
 165                 self.api_version3 = self.config['APIversion'] == 'v3.3' or self.config['APIversion'] == '3'
 166             else:  # get from ending auth_url that end with v3 or with v2.0
 167                 self.api_version3 =  self.url.split("/")[-1] == "v3"
 168             self.session['api_version3'] = self.api_version3
 169             if self.api_version3:
 170                 auth = v3.Password(auth_url=self.url,
 171                                    username=self.user,
 172                                    password=self.passwd,
 173                                    project_name=self.tenant_name,
 174                                    project_id=self.tenant_id,
 175                                    project_domain_id=self.config.get('project_domain_id', 'default'),
 176                                    user_domain_id=self.config.get('user_domain_id', 'default'))
 177             else:
 178                 auth = v2.Password(auth_url=self.url,
 179                                    username=self.user,
 180                                    password=self.passwd,
 181                                    tenant_name=self.tenant_name,
 182                                    tenant_id=self.tenant_id)
 183             sess = session.Session(auth=auth, verify=not self.insecure)
 184             if self.api_version3:
 185                 self.keystone = ksClient_v3.Client(session=sess, endpoint_type=self.endpoint_type)
 186             else:
 187                 self.keystone = ksClient_v2.Client(session=sess, endpoint_type=self.endpoint_type)
 188             self.session['keystone'] = self.keystone
 189             # In order to enable microversion functionality an explicit microversion must be specified in 'config'.
 190             # This implementation approach is due to the warning message in
 191             # https://developer.openstack.org/api-guide/compute/microversions.html
 192             # where it is stated that microversion backwards compatibility is not guaranteed and clients should
 193             # always require an specific microversion.
 194             # To be able to use 'device role tagging' functionality define 'microversion: 2.32' in datacenter config
 195             version = self.config.get("microversion")
 196             if not version:
 197                 version = "2.1"
 198             self.nova = self.session['nova'] = nClient.Client(str(version), session=sess, endpoint_type=self.endpoint_type)
 199             self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess, endpoint_type=self.endpoint_type)
 200             self.cinder = self.session['cinder'] = cClient.Client(2, session=sess, endpoint_type=self.endpoint_type)
 201             if self.endpoint_type == "internalURL":
 202                 glance_service_id = self.keystone.services.list(name="glance")[0].id
 203                 glance_endpoint = self.keystone.endpoints.list(glance_service_id, interface="internal")[0].url
 204             else:
 205                 glance_endpoint = None
 206             self.glance = self.session['glance'] = glClient.Client(2, session=sess, endpoint=glance_endpoint)
 207             #using version 1 of glance client in new_image()
 208             self.glancev1 = self.session['glancev1'] = glClient.Client('1', session=sess,
 209                                                                        endpoint=glance_endpoint)
 210             self.session['reload_client'] = False
 211             self.persistent_info['session'] = self.session
 212             # add availablity zone info inside  self.persistent_info
 213             self._set_availablity_zones()
 214             self.persistent_info['availability_zone'] = self.availability_zone
 215
 216     def __net_os2mano(self, net_list_dict):
 217         '''Transform the net openstack format to mano format
 218         net_list_dict can be a list of dict or a single dict'''
 219         if type(net_list_dict) is dict:
 220             net_list_=(net_list_dict,)
 221         elif type(net_list_dict) is list:
 222             net_list_=net_list_dict
 223         else:
 224             raise TypeError("param net_list_dict must be a list or a dictionary")
 225         for net in net_list_:
 226             if net.get('provider:network_type') == "vlan":
 227                 net['type']='data'
 228             else:
 229                 net['type']='bridge'
 230
 231     def __classification_os2mano(self, class_list_dict):
 232         """Transform the openstack format (Flow Classifier) to mano format
 233         (Classification) class_list_dict can be a list of dict or a single dict
 234         """
 235         if isinstance(class_list_dict, dict):
 236             class_list_ = [class_list_dict]
 237         elif isinstance(class_list_dict, list):
 238             class_list_ = class_list_dict
 239         else:
 240             raise TypeError(
 241                 "param class_list_dict must be a list or a dictionary")
 242         for classification in class_list_:
 243             id = classification.pop('id')
 244             name = classification.pop('name')
 245             description = classification.pop('description')
 246             project_id = classification.pop('project_id')
 247             tenant_id = classification.pop('tenant_id')
 248             original_classification = copy.deepcopy(classification)
 249             classification.clear()
 250             classification['ctype'] = 'legacy_flow_classifier'
 251             classification['definition'] = original_classification
 252             classification['id'] = id
 253             classification['name'] = name
 254             classification['description'] = description
 255             classification['project_id'] = project_id
 256             classification['tenant_id'] = tenant_id
 257
 258     def __sfi_os2mano(self, sfi_list_dict):
 259         """Transform the openstack format (Port Pair) to mano format (SFI)
 260         sfi_list_dict can be a list of dict or a single dict
 261         """
 262         if isinstance(sfi_list_dict, dict):
 263             sfi_list_ = [sfi_list_dict]
 264         elif isinstance(sfi_list_dict, list):
 265             sfi_list_ = sfi_list_dict
 266         else:
 267             raise TypeError(
 268                 "param sfi_list_dict must be a list or a dictionary")
 269         for sfi in sfi_list_:
 270             sfi['ingress_ports'] = []
 271             sfi['egress_ports'] = []
 272             if sfi.get('ingress'):
 273                 sfi['ingress_ports'].append(sfi['ingress'])
 274             if sfi.get('egress'):
 275                 sfi['egress_ports'].append(sfi['egress'])
 276             del sfi['ingress']
 277             del sfi['egress']
 278             params = sfi.get('service_function_parameters')
 279             sfc_encap = False
 280             if params:
 281                 correlation = params.get('correlation')
 282                 if correlation:
 283                     sfc_encap = True
 284             sfi['sfc_encap'] = sfc_encap
 285             del sfi['service_function_parameters']
 286
 287     def __sf_os2mano(self, sf_list_dict):
 288         """Transform the openstack format (Port Pair Group) to mano format (SF)
 289         sf_list_dict can be a list of dict or a single dict
 290         """
 291         if isinstance(sf_list_dict, dict):
 292             sf_list_ = [sf_list_dict]
 293         elif isinstance(sf_list_dict, list):
 294             sf_list_ = sf_list_dict
 295         else:
 296             raise TypeError(
 297                 "param sf_list_dict must be a list or a dictionary")
 298         for sf in sf_list_:
 299             del sf['port_pair_group_parameters']
 300             sf['sfis'] = sf['port_pairs']
 301             del sf['port_pairs']
 302
 303     def __sfp_os2mano(self, sfp_list_dict):
 304         """Transform the openstack format (Port Chain) to mano format (SFP)
 305         sfp_list_dict can be a list of dict or a single dict
 306         """
 307         if isinstance(sfp_list_dict, dict):
 308             sfp_list_ = [sfp_list_dict]
 309         elif isinstance(sfp_list_dict, list):
 310             sfp_list_ = sfp_list_dict
 311         else:
 312             raise TypeError(
 313                 "param sfp_list_dict must be a list or a dictionary")
 314         for sfp in sfp_list_:
 315             params = sfp.pop('chain_parameters')
 316             sfc_encap = False
 317             if params:
 318                 correlation = params.get('correlation')
 319                 if correlation:
 320                     sfc_encap = True
 321             sfp['sfc_encap'] = sfc_encap
 322             sfp['spi'] = sfp.pop('chain_id')
 323             sfp['classifications'] = sfp.pop('flow_classifiers')
 324             sfp['service_functions'] = sfp.pop('port_pair_groups')
 325
 326     # placeholder for now; read TODO note below
 327     def _validate_classification(self, type, definition):
 328         # only legacy_flow_classifier Type is supported at this point
 329         return True
 330         # TODO(igordcard): this method should be an abstract method of an
 331         # abstract Classification class to be implemented by the specific
 332         # Types. Also, abstract vimconnector should call the validation
 333         # method before the implemented VIM connectors are called.
 334
 335     def _format_exception(self, exception):
 336         '''Transform a keystone, nova, neutron  exception into a vimconn exception'''
 337         if isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError,
 338                                   ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed
 339                                   )):
 340             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 341         elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException,
 342                                     neExceptions.NeutronException, nvExceptions.BadRequest)):
 343             raise vimconn.vimconnUnexpectedResponse(type(exception).__name__ + ": " + str(exception))
 344         elif isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound)):
 345             raise vimconn.vimconnNotFoundException(type(exception).__name__ + ": " + str(exception))
 346         elif isinstance(exception, nvExceptions.Conflict):
 347             raise vimconn.vimconnConflictException(type(exception).__name__ + ": " + str(exception))
 348         elif isinstance(exception, vimconn.vimconnException):
 349             raise
 350         else:  # ()
 351             self.logger.error("General Exception " + str(exception), exc_info=True)
 352             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 353
 354     def get_tenant_list(self, filter_dict={}):
 355         '''Obtain tenants of VIM
 356         filter_dict can contain the following keys:
 357             name: filter by tenant name
 358             id: filter by tenant uuid/id
 359             <other VIM specific>
 360         Returns the tenant list of dictionaries: [{'name':'<name>, 'id':'<id>, ...}, ...]
 361         '''
 362         self.logger.debug("Getting tenants from VIM filter: '%s'", str(filter_dict))
 363         try:
 364             self._reload_connection()
 365             if self.api_version3:
 366                 project_class_list = self.keystone.projects.list(name=filter_dict.get("name"))
 367             else:
 368                 project_class_list = self.keystone.tenants.findall(**filter_dict)
 369             project_list=[]
 370             for project in project_class_list:
 371                 if filter_dict.get('id') and filter_dict["id"] != project.id:
 372                     continue
 373                 project_list.append(project.to_dict())
 374             return project_list
 375         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError) as e:
 376             self._format_exception(e)
 377
 378     def new_tenant(self, tenant_name, tenant_description):
 379         '''Adds a new tenant to openstack VIM. Returns the tenant identifier'''
 380         self.logger.debug("Adding a new tenant name: %s", tenant_name)
 381         try:
 382             self._reload_connection()
 383             if self.api_version3:
 384                 project = self.keystone.projects.create(tenant_name, self.config.get("project_domain_id", "default"),
 385                                                         description=tenant_description, is_domain=False)
 386             else:
 387                 project = self.keystone.tenants.create(tenant_name, tenant_description)
 388             return project.id
 389         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 390             self._format_exception(e)
 391
 392     def delete_tenant(self, tenant_id):
 393         '''Delete a tenant from openstack VIM. Returns the old tenant identifier'''
 394         self.logger.debug("Deleting tenant %s from VIM", tenant_id)
 395         try:
 396             self._reload_connection()
 397             if self.api_version3:
 398                 self.keystone.projects.delete(tenant_id)
 399             else:
 400                 self.keystone.tenants.delete(tenant_id)
 401             return tenant_id
 402         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 403             self._format_exception(e)
 404
 405     def new_network(self,net_name, net_type, ip_profile=None, shared=False, vlan=None):
 406         '''Adds a tenant network to VIM. Returns the network identifier'''
 407         self.logger.debug("Adding a new network to VIM name '%s', type '%s'", net_name, net_type)
 408         #self.logger.debug(">>>>>>>>>>>>>>>>>> IP profile %s", str(ip_profile))
 409         try:
 410             new_net = None
 411             self._reload_connection()
 412             network_dict = {'name': net_name, 'admin_state_up': True}
 413             if net_type=="data" or net_type=="ptp":
 414                 if self.config.get('dataplane_physical_net') == None:
 415                     raise vimconn.vimconnConflictException("You must provide a 'dataplane_physical_net' at config value before creating sriov network")
 416                 network_dict["provider:physical_network"] = self.config['dataplane_physical_net'] #"physnet_sriov" #TODO physical
 417                 network_dict["provider:network_type"]     = "vlan"
 418                 if vlan!=None:
 419                     network_dict["provider:network_type"] = vlan
 420
 421                 ####### VIO Specific Changes #########
 422                 if self.vim_type == "VIO":
 423                     if vlan is not None:
 424                         network_dict["provider:segmentation_id"] = vlan
 425                     else:
 426                         if self.config.get('dataplane_net_vlan_range') is None:
 427                             raise vimconn.vimconnConflictException("You must provide "\
 428                                 "'dataplane_net_vlan_range' in format [start_ID - end_ID]"\
 429                                 "at config value before creating sriov network with vlan tag")
 430
 431                         network_dict["provider:segmentation_id"] = self._genrate_vlanID()
 432
 433             network_dict["shared"]=shared
 434             new_net=self.neutron.create_network({'network':network_dict})
 435             #print new_net
 436             #create subnetwork, even if there is no profile
 437             if not ip_profile:
 438                 ip_profile = {}
 439             if 'subnet_address' not in ip_profile:
 440                 #Fake subnet is required
 441                 subnet_rand = random.randint(0, 255)
 442                 ip_profile['subnet_address'] = "192.168.{}.0/24".format(subnet_rand)
 443             if 'ip_version' not in ip_profile:
 444                 ip_profile['ip_version'] = "IPv4"
 445             subnet = {"name":net_name+"-subnet",
 446                     "network_id": new_net["network"]["id"],
 447                     "ip_version": 4 if ip_profile['ip_version']=="IPv4" else 6,
 448                     "cidr": ip_profile['subnet_address']
 449                     }
 450             # Gateway should be set to None if not needed. Otherwise openstack assigns one by default
 451             subnet['gateway_ip'] = ip_profile.get('gateway_address')
 452             if ip_profile.get('dns_address'):
 453                 subnet['dns_nameservers'] = ip_profile['dns_address'].split(";")
 454             if 'dhcp_enabled' in ip_profile:
 455                 subnet['enable_dhcp'] = False if ip_profile['dhcp_enabled']=="false" else True
 456             if 'dhcp_start_address' in ip_profile:
 457                 subnet['allocation_pools'] = []
 458                 subnet['allocation_pools'].append(dict())
 459                 subnet['allocation_pools'][0]['start'] = ip_profile['dhcp_start_address']
 460             if 'dhcp_count' in ip_profile:
 461                 #parts = ip_profile['dhcp_start_address'].split('.')
 462                 #ip_int = (int(parts[0]) << 24) + (int(parts[1]) << 16) + (int(parts[2]) << 8) + int(parts[3])
 463                 ip_int = int(netaddr.IPAddress(ip_profile['dhcp_start_address']))
 464                 ip_int += ip_profile['dhcp_count'] - 1
 465                 ip_str = str(netaddr.IPAddress(ip_int))
 466                 subnet['allocation_pools'][0]['end'] = ip_str
 467             #self.logger.debug(">>>>>>>>>>>>>>>>>> Subnet: %s", str(subnet))
 468             self.neutron.create_subnet({"subnet": subnet} )
 469             return new_net["network"]["id"]
 470         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 471             if new_net:
 472                 self.neutron.delete_network(new_net['network']['id'])
 473             self._format_exception(e)
 474
 475     def get_network_list(self, filter_dict={}):
 476         '''Obtain tenant networks of VIM
 477         Filter_dict can be:
 478             name: network name
 479             id: network uuid
 480             shared: boolean
 481             tenant_id: tenant
 482             admin_state_up: boolean
 483             status: 'ACTIVE'
 484         Returns the network list of dictionaries
 485         '''
 486         self.logger.debug("Getting network from VIM filter: '%s'", str(filter_dict))
 487         try:
 488             self._reload_connection()
 489             if self.api_version3 and "tenant_id" in filter_dict:
 490                 filter_dict['project_id'] = filter_dict.pop('tenant_id') #TODO check
 491             net_dict=self.neutron.list_networks(**filter_dict)
 492             net_list=net_dict["networks"]
 493             self.__net_os2mano(net_list)
 494             return net_list
 495         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 496             self._format_exception(e)
 497
 498     def get_network(self, net_id):
 499         '''Obtain details of network from VIM
 500         Returns the network information from a network id'''
 501         self.logger.debug(" Getting tenant network %s from VIM", net_id)
 502         filter_dict={"id": net_id}
 503         net_list = self.get_network_list(filter_dict)
 504         if len(net_list)==0:
 505             raise vimconn.vimconnNotFoundException("Network '{}' not found".format(net_id))
 506         elif len(net_list)>1:
 507             raise vimconn.vimconnConflictException("Found more than one network with this criteria")
 508         net = net_list[0]
 509         subnets=[]
 510         for subnet_id in net.get("subnets", () ):
 511             try:
 512                 subnet = self.neutron.show_subnet(subnet_id)
 513             except Exception as e:
 514                 self.logger.error("osconnector.get_network(): Error getting subnet %s %s" % (net_id, str(e)))
 515                 subnet = {"id": subnet_id, "fault": str(e)}
 516             subnets.append(subnet)
 517         net["subnets"] = subnets
 518         net["encapsulation"] = net.get('provider:network_type')
 519         net["segmentation_id"] = net.get('provider:segmentation_id')
 520         return net
 521
 522     def delete_network(self, net_id):
 523         '''Deletes a tenant network from VIM. Returns the old network identifier'''
 524         self.logger.debug("Deleting network '%s' from VIM", net_id)
 525         try:
 526             self._reload_connection()
 527             #delete VM ports attached to this networks before the network
 528             ports = self.neutron.list_ports(network_id=net_id)
 529             for p in ports['ports']:
 530                 try:
 531                     self.neutron.delete_port(p["id"])
 532                 except Exception as e:
 533                     self.logger.error("Error deleting port %s: %s", p["id"], str(e))
 534             self.neutron.delete_network(net_id)
 535             return net_id
 536         except (neExceptions.ConnectionFailed, neExceptions.NetworkNotFoundClient, neExceptions.NeutronException,
 537                 ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 538             self._format_exception(e)
 539
 540     def refresh_nets_status(self, net_list):
 541         '''Get the status of the networks
 542            Params: the list of network identifiers
 543            Returns a dictionary with:
 544                 net_id:         #VIM id of this network
 545                     status:     #Mandatory. Text with one of:
 546                                 #  DELETED (not found at vim)
 547                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 548                                 #  OTHER (Vim reported other status not understood)
 549                                 #  ERROR (VIM indicates an ERROR status)
 550                                 #  ACTIVE, INACTIVE, DOWN (admin down),
 551                                 #  BUILD (on building process)
 552                                 #
 553                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 554                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 555
 556         '''
 557         net_dict={}
 558         for net_id in net_list:
 559             net = {}
 560             try:
 561                 net_vim = self.get_network(net_id)
 562                 if net_vim['status'] in netStatus2manoFormat:
 563                     net["status"] = netStatus2manoFormat[ net_vim['status'] ]
 564                 else:
 565                     net["status"] = "OTHER"
 566                     net["error_msg"] = "VIM status reported " + net_vim['status']
 567
 568                 if net['status'] == "ACTIVE" and not net_vim['admin_state_up']:
 569                     net['status'] = 'DOWN'
 570                 try:
 571                     net['vim_info'] = yaml.safe_dump(net_vim, default_flow_style=True, width=256)
 572                 except yaml.representer.RepresenterError:
 573                     net['vim_info'] = str(net_vim)
 574                 if net_vim.get('fault'):  #TODO
 575                     net['error_msg'] = str(net_vim['fault'])
 576             except vimconn.vimconnNotFoundException as e:
 577                 self.logger.error("Exception getting net status: %s", str(e))
 578                 net['status'] = "DELETED"
 579                 net['error_msg'] = str(e)
 580             except vimconn.vimconnException as e:
 581                 self.logger.error("Exception getting net status: %s", str(e))
 582                 net['status'] = "VIM_ERROR"
 583                 net['error_msg'] = str(e)
 584             net_dict[net_id] = net
 585         return net_dict
 586
 587     def get_flavor(self, flavor_id):
 588         '''Obtain flavor details from the  VIM. Returns the flavor dict details'''
 589         self.logger.debug("Getting flavor '%s'", flavor_id)
 590         try:
 591             self._reload_connection()
 592             flavor = self.nova.flavors.find(id=flavor_id)
 593             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 594             return flavor.to_dict()
 595         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 596             self._format_exception(e)
 597
 598     def get_flavor_id_from_data(self, flavor_dict):
 599         """Obtain flavor id that match the flavor description
 600            Returns the flavor_id or raises a vimconnNotFoundException
 601            flavor_dict: contains the required ram, vcpus, disk
 602            If 'use_existing_flavors' is set to True at config, the closer flavor that provides same or more ram, vcpus
 603                 and disk is returned. Otherwise a flavor with exactly same ram, vcpus and disk is returned or a
 604                 vimconnNotFoundException is raised
 605         """
 606         exact_match = False if self.config.get('use_existing_flavors') else True
 607         try:
 608             self._reload_connection()
 609             flavor_candidate_id = None
 610             flavor_candidate_data = (10000, 10000, 10000)
 611             flavor_target = (flavor_dict["ram"], flavor_dict["vcpus"], flavor_dict["disk"])
 612             # numa=None
 613             numas = flavor_dict.get("extended", {}).get("numas")
 614             if numas:
 615                 #TODO
 616                 raise vimconn.vimconnNotFoundException("Flavor with EPA still not implemted")
 617                 # if len(numas) > 1:
 618                 #     raise vimconn.vimconnNotFoundException("Cannot find any flavor with more than one numa")
 619                 # numa=numas[0]
 620                 # numas = extended.get("numas")
 621             for flavor in self.nova.flavors.list():
 622                 epa = flavor.get_keys()
 623                 if epa:
 624                     continue
 625                     # TODO
 626                 flavor_data = (flavor.ram, flavor.vcpus, flavor.disk)
 627                 if flavor_data == flavor_target:
 628                     return flavor.id
 629                 elif not exact_match and flavor_target < flavor_data < flavor_candidate_data:
 630                     flavor_candidate_id = flavor.id
 631                     flavor_candidate_data = flavor_data
 632             if not exact_match and flavor_candidate_id:
 633                 return flavor_candidate_id
 634             raise vimconn.vimconnNotFoundException("Cannot find any flavor matching '{}'".format(str(flavor_dict)))
 635         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 636             self._format_exception(e)
 637
 638
 639     def new_flavor(self, flavor_data, change_name_if_used=True):
 640         '''Adds a tenant flavor to openstack VIM
 641         if change_name_if_used is True, it will change name in case of conflict, because it is not supported name repetition
 642         Returns the flavor identifier
 643         '''
 644         self.logger.debug("Adding flavor '%s'", str(flavor_data))
 645         retry=0
 646         max_retries=3
 647         name_suffix = 0
 648         name=flavor_data['name']
 649         while retry<max_retries:
 650             retry+=1
 651             try:
 652                 self._reload_connection()
 653                 if change_name_if_used:
 654                     #get used names
 655                     fl_names=[]
 656                     fl=self.nova.flavors.list()
 657                     for f in fl:
 658                         fl_names.append(f.name)
 659                     while name in fl_names:
 660                         name_suffix += 1
 661                         name = flavor_data['name']+"-" + str(name_suffix)
 662
 663                 ram = flavor_data.get('ram',64)
 664                 vcpus = flavor_data.get('vcpus',1)
 665                 numa_properties=None
 666
 667                 extended = flavor_data.get("extended")
 668                 if extended:
 669                     numas=extended.get("numas")
 670                     if numas:
 671                         numa_nodes = len(numas)
 672                         if numa_nodes > 1:
 673                             return -1, "Can not add flavor with more than one numa"
 674                         numa_properties = {"hw:numa_nodes":str(numa_nodes)}
 675                         numa_properties["hw:mem_page_size"] = "large"
 676                         numa_properties["hw:cpu_policy"] = "dedicated"
 677                         numa_properties["hw:numa_mempolicy"] = "strict"
 678                         if self.vim_type == "VIO":
 679                             numa_properties["vmware:extra_config"] = '{"numa.nodeAffinity":"0"}'
 680                             numa_properties["vmware:latency_sensitivity_level"] = "high"
 681                         for numa in numas:
 682                             #overwrite ram and vcpus
 683                             ram = numa['memory']*1024
 684                             #See for reference: https://specs.openstack.org/openstack/nova-specs/specs/mitaka/implemented/virt-driver-cpu-thread-pinning.html
 685                             if 'paired-threads' in numa:
 686                                 vcpus = numa['paired-threads']*2
 687                                 #cpu_thread_policy "require" implies that the compute node must have an STM architecture
 688                                 numa_properties["hw:cpu_thread_policy"] = "require"
 689                                 numa_properties["hw:cpu_policy"] = "dedicated"
 690                             elif 'cores' in numa:
 691                                 vcpus = numa['cores']
 692                                 # cpu_thread_policy "prefer" implies that the host must not have an SMT architecture, or a non-SMT architecture will be emulated
 693                                 numa_properties["hw:cpu_thread_policy"] = "isolate"
 694                                 numa_properties["hw:cpu_policy"] = "dedicated"
 695                             elif 'threads' in numa:
 696                                 vcpus = numa['threads']
 697                                 # cpu_thread_policy "prefer" implies that the host may or may not have an SMT architecture
 698                                 numa_properties["hw:cpu_thread_policy"] = "prefer"
 699                                 numa_properties["hw:cpu_policy"] = "dedicated"
 700                             # for interface in numa.get("interfaces",() ):
 701                             #     if interface["dedicated"]=="yes":
 702                             #         raise vimconn.vimconnException("Passthrough interfaces are not supported for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable)
 703                             #     #TODO, add the key 'pci_passthrough:alias"="<label at config>:<number ifaces>"' when a way to connect it is available
 704
 705                 #create flavor
 706                 new_flavor=self.nova.flavors.create(name,
 707                                 ram,
 708                                 vcpus,
 709                                 flavor_data.get('disk',1),
 710                                 is_public=flavor_data.get('is_public', True)
 711                             )
 712                 #add metadata
 713                 if numa_properties:
 714                     new_flavor.set_keys(numa_properties)
 715                 return new_flavor.id
 716             except nvExceptions.Conflict as e:
 717                 if change_name_if_used and retry < max_retries:
 718                     continue
 719                 self._format_exception(e)
 720             #except nvExceptions.BadRequest as e:
 721             except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 722                 self._format_exception(e)
 723
 724     def delete_flavor(self,flavor_id):
 725         '''Deletes a tenant flavor from openstack VIM. Returns the old flavor_id
 726         '''
 727         try:
 728             self._reload_connection()
 729             self.nova.flavors.delete(flavor_id)
 730             return flavor_id
 731         #except nvExceptions.BadRequest as e:
 732         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 733             self._format_exception(e)
 734
 735     def new_image(self,image_dict):
 736         '''
 737         Adds a tenant image to VIM. imge_dict is a dictionary with:
 738             name: name
 739             disk_format: qcow2, vhd, vmdk, raw (by default), ...
 740             location: path or URI
 741             public: "yes" or "no"
 742             metadata: metadata of the image
 743         Returns the image_id
 744         '''
 745         retry=0
 746         max_retries=3
 747         while retry<max_retries:
 748             retry+=1
 749             try:
 750                 self._reload_connection()
 751                 #determine format  http://docs.openstack.org/developer/glance/formats.html
 752                 if "disk_format" in image_dict:
 753                     disk_format=image_dict["disk_format"]
 754                 else: #autodiscover based on extension
 755                     if image_dict['location'][-6:]==".qcow2":
 756                         disk_format="qcow2"
 757                     elif image_dict['location'][-4:]==".vhd":
 758                         disk_format="vhd"
 759                     elif image_dict['location'][-5:]==".vmdk":
 760                         disk_format="vmdk"
 761                     elif image_dict['location'][-4:]==".vdi":
 762                         disk_format="vdi"
 763                     elif image_dict['location'][-4:]==".iso":
 764                         disk_format="iso"
 765                     elif image_dict['location'][-4:]==".aki":
 766                         disk_format="aki"
 767                     elif image_dict['location'][-4:]==".ari":
 768                         disk_format="ari"
 769                     elif image_dict['location'][-4:]==".ami":
 770                         disk_format="ami"
 771                     else:
 772                         disk_format="raw"
 773                 self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location'])
 774                 if image_dict['location'][0:4]=="http":
 775                     new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 776                             container_format="bare", location=image_dict['location'], disk_format=disk_format)
 777                 else: #local path
 778                     with open(image_dict['location']) as fimage:
 779                         new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 780                             container_format="bare", data=fimage, disk_format=disk_format)
 781                 #insert metadata. We cannot use 'new_image.properties.setdefault'
 782                 #because nova and glance are "INDEPENDENT" and we are using nova for reading metadata
 783                 new_image_nova=self.nova.images.find(id=new_image.id)
 784                 new_image_nova.metadata.setdefault('location',image_dict['location'])
 785                 metadata_to_load = image_dict.get('metadata')
 786                 if metadata_to_load:
 787                     for k,v in yaml.load(metadata_to_load).iteritems():
 788                         new_image_nova.metadata.setdefault(k,v)
 789                 return new_image.id
 790             except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e:
 791                 self._format_exception(e)
 792             except (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 793                 if retry==max_retries:
 794                     continue
 795                 self._format_exception(e)
 796             except IOError as e:  #can not open the file
 797                 raise vimconn.vimconnConnectionException(type(e).__name__ + ": " + str(e)+ " for " + image_dict['location'],
 798                                                          http_code=vimconn.HTTP_Bad_Request)
 799
 800     def delete_image(self, image_id):
 801         '''Deletes a tenant image from openstack VIM. Returns the old id
 802         '''
 803         try:
 804             self._reload_connection()
 805             self.nova.images.delete(image_id)
 806             return image_id
 807         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e: #TODO remove
 808             self._format_exception(e)
 809
 810     def get_image_id_from_path(self, path):
 811         '''Get the image id from image path in the VIM database. Returns the image_id'''
 812         try:
 813             self._reload_connection()
 814             images = self.nova.images.list()
 815             for image in images:
 816                 if image.metadata.get("location")==path:
 817                     return image.id
 818             raise vimconn.vimconnNotFoundException("image with location '{}' not found".format( path))
 819         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 820             self._format_exception(e)
 821
 822     def get_image_list(self, filter_dict={}):
 823         '''Obtain tenant images from VIM
 824         Filter_dict can be:
 825             id: image id
 826             name: image name
 827             checksum: image checksum
 828         Returns the image list of dictionaries:
 829             [{<the fields at Filter_dict plus some VIM specific>}, ...]
 830             List can be empty
 831         '''
 832         self.logger.debug("Getting image list from VIM filter: '%s'", str(filter_dict))
 833         try:
 834             self._reload_connection()
 835             filter_dict_os=filter_dict.copy()
 836             #First we filter by the available filter fields: name, id. The others are removed.
 837             filter_dict_os.pop('checksum',None)
 838             image_list=self.nova.images.findall(**filter_dict_os)
 839             if len(image_list)==0:
 840                 return []
 841             #Then we filter by the rest of filter fields: checksum
 842             filtered_list = []
 843             for image in image_list:
 844                 image_class=self.glance.images.get(image.id)
 845                 if 'checksum' not in filter_dict or image_class['checksum']==filter_dict.get('checksum'):
 846                     filtered_list.append(image_class.copy())
 847             return filtered_list
 848         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 849             self._format_exception(e)
 850
 851     @staticmethod
 852     def _create_mimemultipart(content_list):
 853         """Creates a MIMEmultipart text combining the content_list
 854         :param content_list: list of text scripts to be combined
 855         :return: str of the created MIMEmultipart. If the list is empty returns None, if the list contains only one
 856         element MIMEmultipart is not created and this content is returned
 857         """
 858         if not content_list:
 859             return None
 860         elif len(content_list) == 1:
 861             return content_list[0]
 862         combined_message = MIMEMultipart()
 863         for content in content_list:
 864             if content.startswith('#include'):
 865                 format = 'text/x-include-url'
 866             elif content.startswith('#include-once'):
 867                 format = 'text/x-include-once-url'
 868             elif content.startswith('#!'):
 869                 format = 'text/x-shellscript'
 870             elif content.startswith('#cloud-config'):
 871                 format = 'text/cloud-config'
 872             elif content.startswith('#cloud-config-archive'):
 873                 format = 'text/cloud-config-archive'
 874             elif content.startswith('#upstart-job'):
 875                 format = 'text/upstart-job'
 876             elif content.startswith('#part-handler'):
 877                 format = 'text/part-handler'
 878             elif content.startswith('#cloud-boothook'):
 879                 format = 'text/cloud-boothook'
 880             else:  # by default
 881                 format = 'text/x-shellscript'
 882             sub_message = MIMEText(content, format, sys.getdefaultencoding())
 883             combined_message.attach(sub_message)
 884         return combined_message.as_string()
 885
 886     def __wait_for_vm(self, vm_id, status):
 887         """wait until vm is in the desired status and return True.
 888         If the VM gets in ERROR status, return false.
 889         If the timeout is reached generate an exception"""
 890         elapsed_time = 0
 891         while elapsed_time < server_timeout:
 892             vm_status = self.nova.servers.get(vm_id).status
 893             if vm_status == status:
 894                 return True
 895             if vm_status == 'ERROR':
 896                 return False
 897             time.sleep(1)
 898             elapsed_time += 1
 899
 900         # if we exceeded the timeout rollback
 901         if elapsed_time >= server_timeout:
 902             raise vimconn.vimconnException('Timeout waiting for instance ' + vm_id + ' to get ' + status,
 903                                            http_code=vimconn.HTTP_Request_Timeout)
 904
 905     def _get_openstack_availablity_zones(self):
 906         """
 907         Get from openstack availability zones available
 908         :return:
 909         """
 910         try:
 911             openstack_availability_zone = self.nova.availability_zones.list()
 912             openstack_availability_zone = [str(zone.zoneName) for zone in openstack_availability_zone
 913                                            if zone.zoneName != 'internal']
 914             return openstack_availability_zone
 915         except Exception as e:
 916             return None
 917
 918     def _set_availablity_zones(self):
 919         """
 920         Set vim availablity zone
 921         :return:
 922         """
 923
 924         if 'availability_zone' in self.config:
 925             vim_availability_zones = self.config.get('availability_zone')
 926             if isinstance(vim_availability_zones, str):
 927                 self.availability_zone = [vim_availability_zones]
 928             elif isinstance(vim_availability_zones, list):
 929                 self.availability_zone = vim_availability_zones
 930         else:
 931             self.availability_zone = self._get_openstack_availablity_zones()
 932
 933     def _get_vm_availability_zone(self, availability_zone_index, availability_zone_list):
 934         """
 935         Return thge availability zone to be used by the created VM.
 936         :return: The VIM availability zone to be used or None
 937         """
 938         if availability_zone_index is None:
 939             if not self.config.get('availability_zone'):
 940                 return None
 941             elif isinstance(self.config.get('availability_zone'), str):
 942                 return self.config['availability_zone']
 943             else:
 944                 # TODO consider using a different parameter at config for default AV and AV list match
 945                 return self.config['availability_zone'][0]
 946
 947         vim_availability_zones = self.availability_zone
 948         # check if VIM offer enough availability zones describe in the VNFD
 949         if vim_availability_zones and len(availability_zone_list) <= len(vim_availability_zones):
 950             # check if all the names of NFV AV match VIM AV names
 951             match_by_index = False
 952             for av in availability_zone_list:
 953                 if av not in vim_availability_zones:
 954                     match_by_index = True
 955                     break
 956             if match_by_index:
 957                 return vim_availability_zones[availability_zone_index]
 958             else:
 959                 return availability_zone_list[availability_zone_index]
 960         else:
 961             raise vimconn.vimconnConflictException("No enough availability zones at VIM for this deployment")
 962
 963     def new_vminstance(self, name, description, start, image_id, flavor_id, net_list, cloud_config=None, disk_list=None,
 964                        availability_zone_index=None, availability_zone_list=None):
 965         '''Adds a VM instance to VIM
 966         Params:
 967             start: indicates if VM must start or boot in pause mode. Ignored
 968             image_id,flavor_id: iamge and flavor uuid
 969             net_list: list of interfaces, each one is a dictionary with:
 970                 name:
 971                 net_id: network uuid to connect
 972                 vpci: virtual vcpi to assign, ignored because openstack lack #TODO
 973                 model: interface model, ignored #TODO
 974                 mac_address: used for  SR-IOV ifaces #TODO for other types
 975                 use: 'data', 'bridge',  'mgmt'
 976                 type: 'virtual', 'PF', 'VF', 'VFnotShared'
 977                 vim_id: filled/added by this function
 978                 floating_ip: True/False (or it can be None)
 979                 'cloud_config': (optional) dictionary with:
 980                 'key-pairs': (optional) list of strings with the public key to be inserted to the default user
 981                 'users': (optional) list of users to be inserted, each item is a dict with:
 982                     'name': (mandatory) user name,
 983                     'key-pairs': (optional) list of strings with the public key to be inserted to the user
 984                 'user-data': (optional) string is a text script to be passed directly to cloud-init
 985                 'config-files': (optional). List of files to be transferred. Each item is a dict with:
 986                     'dest': (mandatory) string with the destination absolute path
 987                     'encoding': (optional, by default text). Can be one of:
 988                         'b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64'
 989                     'content' (mandatory): string with the content of the file
 990                     'permissions': (optional) string with file permissions, typically octal notation '0644'
 991                     'owner': (optional) file owner, string with the format 'owner:group'
 992                 'boot-data-drive': boolean to indicate if user-data must be passed using a boot drive (hard disk)
 993             'disk_list': (optional) list with additional disks to the VM. Each item is a dict with:
 994                 'image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted
 995                 'size': (mandatory) string with the size of the disk in GB
 996             availability_zone_index: Index of availability_zone_list to use for this this VM. None if not AV required
 997             availability_zone_list: list of availability zones given by user in the VNFD descriptor.  Ignore if
 998                 availability_zone_index is None
 999                 #TODO ip, security groups
1000         Returns the instance identifier
1001         '''
1002         self.logger.debug("new_vminstance input: image='%s' flavor='%s' nics='%s'",image_id, flavor_id,str(net_list))
1003         try:
1004             server = None
1005             metadata={}
1006             net_list_vim=[]
1007             external_network=[]     # list of external networks to be connected to instance, later on used to create floating_ip
1008             no_secured_ports = []   # List of port-is with port-security disabled
1009             self._reload_connection()
1010             metadata_vpci={}   # For a specific neutron plugin
1011             block_device_mapping = None
1012             for net in net_list:
1013                 if not net.get("net_id"): #skip non connected iface
1014                     continue
1015
1016                 port_dict={
1017                     "network_id": net["net_id"],
1018                     "name": net.get("name"),
1019                     "admin_state_up": True
1020                 }
1021                 if net["type"]=="virtual":
1022                     if "vpci" in net:
1023                         metadata_vpci[ net["net_id"] ] = [[ net["vpci"], "" ]]
1024                 elif net["type"]=="VF": # for VF
1025                     if "vpci" in net:
1026                         if "VF" not in metadata_vpci:
1027                             metadata_vpci["VF"]=[]
1028                         metadata_vpci["VF"].append([ net["vpci"], "" ])
1029                     port_dict["binding:vnic_type"]="direct"
1030                     ########## VIO specific Changes #######
1031                     if self.vim_type == "VIO":
1032                         #Need to create port with port_security_enabled = False and no-security-groups
1033                         port_dict["port_security_enabled"]=False
1034                         port_dict["provider_security_groups"]=[]
1035                         port_dict["security_groups"]=[]
1036                 else: #For PT
1037                     ########## VIO specific Changes #######
1038                     #Current VIO release does not support port with type 'direct-physical'
1039                     #So no need to create virtual port in case of PCI-device.
1040                     #Will update port_dict code when support gets added in next VIO release
1041                     if self.vim_type == "VIO":
1042                         raise vimconn.vimconnNotSupportedException("Current VIO release does not support full passthrough (PT)")
1043                     if "vpci" in net:
1044                         if "PF" not in metadata_vpci:
1045                             metadata_vpci["PF"]=[]
1046                         metadata_vpci["PF"].append([ net["vpci"], "" ])
1047                     port_dict["binding:vnic_type"]="direct-physical"
1048                 if not port_dict["name"]:
1049                     port_dict["name"]=name
1050                 if net.get("mac_address"):
1051                     port_dict["mac_address"]=net["mac_address"]
1052                 new_port = self.neutron.create_port({"port": port_dict })
1053                 net["mac_adress"] = new_port["port"]["mac_address"]
1054                 net["vim_id"] = new_port["port"]["id"]
1055                 # if try to use a network without subnetwork, it will return a emtpy list
1056                 fixed_ips = new_port["port"].get("fixed_ips")
1057                 if fixed_ips:
1058                     net["ip"] = fixed_ips[0].get("ip_address")
1059                 else:
1060                     net["ip"] = None
1061
1062                 port = {"port-id": new_port["port"]["id"]}
1063                 if float(self.nova.api_version.get_string()) >= 2.32:
1064                     port["tag"] = new_port["port"]["name"]
1065                 net_list_vim.append(port)
1066
1067                 if net.get('floating_ip', False):
1068                     net['exit_on_floating_ip_error'] = True
1069                     external_network.append(net)
1070                 elif net['use'] == 'mgmt' and self.config.get('use_floating_ip'):
1071                     net['exit_on_floating_ip_error'] = False
1072                     external_network.append(net)
1073
1074                 # If port security is disabled when the port has not yet been attached to the VM, then all vm traffic is dropped.
1075                 # As a workaround we wait until the VM is active and then disable the port-security
1076                 if net.get("port_security") == False:
1077                     no_secured_ports.append(new_port["port"]["id"])
1078
1079             if metadata_vpci:
1080                 metadata = {"pci_assignement": json.dumps(metadata_vpci)}
1081                 if len(metadata["pci_assignement"]) >255:
1082                     #limit the metadata size
1083                     #metadata["pci_assignement"] = metadata["pci_assignement"][0:255]
1084                     self.logger.warn("Metadata deleted since it exceeds the expected length (255) ")
1085                     metadata = {}
1086
1087             self.logger.debug("name '%s' image_id '%s'flavor_id '%s' net_list_vim '%s' description '%s' metadata %s",
1088                               name, image_id, flavor_id, str(net_list_vim), description, str(metadata))
1089
1090             security_groups   = self.config.get('security_groups')
1091             if type(security_groups) is str:
1092                 security_groups = ( security_groups, )
1093             #cloud config
1094             userdata=None
1095             config_drive = None
1096             userdata_list = []
1097             if isinstance(cloud_config, dict):
1098                 if cloud_config.get("user-data"):
1099                     if isinstance(cloud_config["user-data"], str):
1100                         userdata_list.append(cloud_config["user-data"])
1101                     else:
1102                         for u in cloud_config["user-data"]:
1103                             userdata_list.append(u)
1104                 if cloud_config.get("boot-data-drive") != None:
1105                     config_drive = cloud_config["boot-data-drive"]
1106                 if cloud_config.get("config-files") or cloud_config.get("users") or cloud_config.get("key-pairs"):
1107                     userdata_dict={}
1108                     #default user
1109                     if cloud_config.get("key-pairs"):
1110                         userdata_dict["ssh-authorized-keys"] = cloud_config["key-pairs"]
1111                         userdata_dict["users"] = [{"default": None, "ssh-authorized-keys": cloud_config["key-pairs"] }]
1112                     if cloud_config.get("users"):
1113                         if "users" not in userdata_dict:
1114                             userdata_dict["users"] = [ "default" ]
1115                         for user in cloud_config["users"]:
1116                             user_info = {
1117                                 "name" : user["name"],
1118                                 "sudo": "ALL = (ALL)NOPASSWD:ALL"
1119                             }
1120                             if "user-info" in user:
1121                                 user_info["gecos"] = user["user-info"]
1122                             if user.get("key-pairs"):
1123                                 user_info["ssh-authorized-keys"] = user["key-pairs"]
1124                             userdata_dict["users"].append(user_info)
1125
1126                     if cloud_config.get("config-files"):
1127                         userdata_dict["write_files"] = []
1128                         for file in cloud_config["config-files"]:
1129                             file_info = {
1130                                 "path" : file["dest"],
1131                                 "content": file["content"]
1132                             }
1133                             if file.get("encoding"):
1134                                 file_info["encoding"] = file["encoding"]
1135                             if file.get("permissions"):
1136                                 file_info["permissions"] = file["permissions"]
1137                             if file.get("owner"):
1138                                 file_info["owner"] = file["owner"]
1139                             userdata_dict["write_files"].append(file_info)
1140                     userdata_list.append("#cloud-config\n" + yaml.safe_dump(userdata_dict, indent=4,
1141                                                                               default_flow_style=False))
1142                     userdata = self._create_mimemultipart(userdata_list)
1143                 self.logger.debug("userdata: %s", userdata)
1144             elif isinstance(cloud_config, str):
1145                 userdata = cloud_config
1146
1147             #Create additional volumes in case these are present in disk_list
1148             base_disk_index = ord('b')
1149             if disk_list != None:
1150                 block_device_mapping = {}
1151                 for disk in disk_list:
1152                     if 'image_id' in disk:
1153                         volume = self.cinder.volumes.create(size = disk['size'],name = name + '_vd' +
1154                                     chr(base_disk_index), imageRef = disk['image_id'])
1155                     else:
1156                         volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1157                                     chr(base_disk_index))
1158                     block_device_mapping['_vd' +  chr(base_disk_index)] = volume.id
1159                     base_disk_index += 1
1160
1161                 #wait until volumes are with status available
1162                 keep_waiting = True
1163                 elapsed_time = 0
1164                 while keep_waiting and elapsed_time < volume_timeout:
1165                     keep_waiting = False
1166                     for volume_id in block_device_mapping.itervalues():
1167                         if self.cinder.volumes.get(volume_id).status != 'available':
1168                             keep_waiting = True
1169                     if keep_waiting:
1170                         time.sleep(1)
1171                         elapsed_time += 1
1172
1173                 #if we exceeded the timeout rollback
1174                 if elapsed_time >= volume_timeout:
1175                     #delete the volumes we just created
1176                     for volume_id in block_device_mapping.itervalues():
1177                         self.cinder.volumes.delete(volume_id)
1178
1179                     #delete ports we just created
1180                     for net_item  in net_list_vim:
1181                         if 'port-id' in net_item:
1182                             self.neutron.delete_port(net_item['port-id'])
1183
1184                     raise vimconn.vimconnException('Timeout creating volumes for instance ' + name,
1185                                                    http_code=vimconn.HTTP_Request_Timeout)
1186             # get availability Zone
1187             vm_av_zone = self._get_vm_availability_zone(availability_zone_index, availability_zone_list)
1188
1189             self.logger.debug("nova.servers.create({}, {}, {}, nics={}, meta={}, security_groups={}, "
1190                               "availability_zone={}, key_name={}, userdata={}, config_drive={}, "
1191                               "block_device_mapping={})".format(name, image_id, flavor_id, net_list_vim, metadata,
1192                                                                 security_groups, vm_av_zone, self.config.get('keypair'),
1193                               userdata, config_drive, block_device_mapping))
1194             server = self.nova.servers.create(name, image_id, flavor_id, nics=net_list_vim, meta=metadata,
1195                                               security_groups=security_groups,
1196                                               availability_zone=vm_av_zone,
1197                                               key_name=self.config.get('keypair'),
1198                                               userdata=userdata,
1199                                               config_drive=config_drive,
1200                                               block_device_mapping=block_device_mapping
1201                                               )  # , description=description)
1202
1203             # Previously mentioned workaround to wait until the VM is active and then disable the port-security
1204             if no_secured_ports:
1205                 self.__wait_for_vm(server.id, 'ACTIVE')
1206
1207             for port_id in no_secured_ports:
1208                 try:
1209                     self.neutron.update_port(port_id, {"port": {"port_security_enabled": False, "security_groups": None} })
1210
1211                 except Exception as e:
1212                     self.logger.error("It was not possible to disable port security for port {}".format(port_id))
1213                     self.delete_vminstance(server.id)
1214                     raise
1215
1216             #print "DONE :-)", server
1217             pool_id = None
1218             floating_ips = self.neutron.list_floatingips().get("floatingips", ())
1219
1220             if external_network:
1221                 self.__wait_for_vm(server.id, 'ACTIVE')
1222
1223             for floating_network in external_network:
1224                 try:
1225                     assigned = False
1226                     while(assigned == False):
1227                         if floating_ips:
1228                             ip = floating_ips.pop(0)
1229                             if not ip.get("port_id", False) and ip.get('tenant_id') == server.tenant_id:
1230                                 free_floating_ip = ip.get("floating_ip_address")
1231                                 try:
1232                                     fix_ip = floating_network.get('ip')
1233                                     server.add_floating_ip(free_floating_ip, fix_ip)
1234                                     assigned = True
1235                                 except Exception as e:
1236                                     raise vimconn.vimconnException(type(e).__name__ + ": Cannot create floating_ip "+  str(e), http_code=vimconn.HTTP_Conflict)
1237                         else:
1238                             #Find the external network
1239                             external_nets = list()
1240                             for net in self.neutron.list_networks()['networks']:
1241                                 if net['router:external']:
1242                                         external_nets.append(net)
1243
1244                             if len(external_nets) == 0:
1245                                 raise vimconn.vimconnException("Cannot create floating_ip automatically since no external "
1246                                                                "network is present",
1247                                                                 http_code=vimconn.HTTP_Conflict)
1248                             if len(external_nets) > 1:
1249                                 raise vimconn.vimconnException("Cannot create floating_ip automatically since multiple "
1250                                                                "external networks are present",
1251                                                                http_code=vimconn.HTTP_Conflict)
1252
1253                             pool_id = external_nets[0].get('id')
1254                             param = {'floatingip': {'floating_network_id': pool_id, 'tenant_id': server.tenant_id}}
1255                             try:
1256                                 #self.logger.debug("Creating floating IP")
1257                                 new_floating_ip = self.neutron.create_floatingip(param)
1258                                 free_floating_ip = new_floating_ip['floatingip']['floating_ip_address']
1259                                 fix_ip = floating_network.get('ip')
1260                                 server.add_floating_ip(free_floating_ip, fix_ip)
1261                                 assigned=True
1262                             except Exception as e:
1263                                 raise vimconn.vimconnException(type(e).__name__ + ": Cannot assign floating_ip "+  str(e), http_code=vimconn.HTTP_Conflict)
1264                 except Exception as e:
1265                     if not floating_network['exit_on_floating_ip_error']:
1266                         self.logger.warn("Cannot create floating_ip. %s", str(e))
1267                         continue
1268                     raise
1269
1270             return server.id
1271 #        except nvExceptions.NotFound as e:
1272 #            error_value=-vimconn.HTTP_Not_Found
1273 #            error_text= "vm instance %s not found" % vm_id
1274 #        except TypeError as e:
1275 #            raise vimconn.vimconnException(type(e).__name__ + ": "+  str(e), http_code=vimconn.HTTP_Bad_Request)
1276
1277         except Exception as e:
1278             # delete the volumes we just created
1279             if block_device_mapping:
1280                 for volume_id in block_device_mapping.itervalues():
1281                     self.cinder.volumes.delete(volume_id)
1282
1283             # Delete the VM
1284             if server != None:
1285                 self.delete_vminstance(server.id)
1286             else:
1287                 # delete ports we just created
1288                 for net_item in net_list_vim:
1289                     if 'port-id' in net_item:
1290                         self.neutron.delete_port(net_item['port-id'])
1291
1292             self._format_exception(e)
1293
1294     def get_vminstance(self,vm_id):
1295         '''Returns the VM instance information from VIM'''
1296         #self.logger.debug("Getting VM from VIM")
1297         try:
1298             self._reload_connection()
1299             server = self.nova.servers.find(id=vm_id)
1300             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
1301             return server.to_dict()
1302         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1303             self._format_exception(e)
1304
1305     def get_vminstance_console(self,vm_id, console_type="vnc"):
1306         '''
1307         Get a console for the virtual machine
1308         Params:
1309             vm_id: uuid of the VM
1310             console_type, can be:
1311                 "novnc" (by default), "xvpvnc" for VNC types,
1312                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
1313         Returns dict with the console parameters:
1314                 protocol: ssh, ftp, http, https, ...
1315                 server:   usually ip address
1316                 port:     the http, ssh, ... port
1317                 suffix:   extra text, e.g. the http path and query string
1318         '''
1319         self.logger.debug("Getting VM CONSOLE from VIM")
1320         try:
1321             self._reload_connection()
1322             server = self.nova.servers.find(id=vm_id)
1323             if console_type == None or console_type == "novnc":
1324                 console_dict = server.get_vnc_console("novnc")
1325             elif console_type == "xvpvnc":
1326                 console_dict = server.get_vnc_console(console_type)
1327             elif console_type == "rdp-html5":
1328                 console_dict = server.get_rdp_console(console_type)
1329             elif console_type == "spice-html5":
1330                 console_dict = server.get_spice_console(console_type)
1331             else:
1332                 raise vimconn.vimconnException("console type '{}' not allowed".format(console_type), http_code=vimconn.HTTP_Bad_Request)
1333
1334             console_dict1 = console_dict.get("console")
1335             if console_dict1:
1336                 console_url = console_dict1.get("url")
1337                 if console_url:
1338                     #parse console_url
1339                     protocol_index = console_url.find("//")
1340                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1341                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1342                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1343                         return -vimconn.HTTP_Internal_Server_Error, "Unexpected response from VIM"
1344                     console_dict={"protocol": console_url[0:protocol_index],
1345                                   "server":   console_url[protocol_index+2:port_index],
1346                                   "port":     console_url[port_index:suffix_index],
1347                                   "suffix":   console_url[suffix_index+1:]
1348                                   }
1349                     protocol_index += 2
1350                     return console_dict
1351             raise vimconn.vimconnUnexpectedResponse("Unexpected response from VIM")
1352
1353         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.BadRequest, ConnectionError) as e:
1354             self._format_exception(e)
1355
1356     def delete_vminstance(self, vm_id):
1357         '''Removes a VM instance from VIM. Returns the old identifier
1358         '''
1359         #print "osconnector: Getting VM from VIM"
1360         try:
1361             self._reload_connection()
1362             #delete VM ports attached to this networks before the virtual machine
1363             ports = self.neutron.list_ports(device_id=vm_id)
1364             for p in ports['ports']:
1365                 try:
1366                     self.neutron.delete_port(p["id"])
1367                 except Exception as e:
1368                     self.logger.error("Error deleting port: " + type(e).__name__ + ": "+  str(e))
1369
1370             #commented because detaching the volumes makes the servers.delete not work properly ?!?
1371             #dettach volumes attached
1372             server = self.nova.servers.get(vm_id)
1373             volumes_attached_dict = server._info['os-extended-volumes:volumes_attached']
1374             #for volume in volumes_attached_dict:
1375             #    self.cinder.volumes.detach(volume['id'])
1376
1377             self.nova.servers.delete(vm_id)
1378
1379             #delete volumes.
1380             #Although having detached them should have them  in active status
1381             #we ensure in this loop
1382             keep_waiting = True
1383             elapsed_time = 0
1384             while keep_waiting and elapsed_time < volume_timeout:
1385                 keep_waiting = False
1386                 for volume in volumes_attached_dict:
1387                     if self.cinder.volumes.get(volume['id']).status != 'available':
1388                         keep_waiting = True
1389                     else:
1390                         self.cinder.volumes.delete(volume['id'])
1391                 if keep_waiting:
1392                     time.sleep(1)
1393                     elapsed_time += 1
1394
1395             return vm_id
1396         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
1397             self._format_exception(e)
1398         #TODO insert exception vimconn.HTTP_Unauthorized
1399         #if reaching here is because an exception
1400
1401     def refresh_vms_status(self, vm_list):
1402         '''Get the status of the virtual machines and their interfaces/ports
1403            Params: the list of VM identifiers
1404            Returns a dictionary with:
1405                 vm_id:          #VIM id of this Virtual Machine
1406                     status:     #Mandatory. Text with one of:
1407                                 #  DELETED (not found at vim)
1408                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1409                                 #  OTHER (Vim reported other status not understood)
1410                                 #  ERROR (VIM indicates an ERROR status)
1411                                 #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
1412                                 #  CREATING (on building process), ERROR
1413                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
1414                                 #
1415                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
1416                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
1417                     interfaces:
1418                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
1419                         mac_address:      #Text format XX:XX:XX:XX:XX:XX
1420                         vim_net_id:       #network id where this interface is connected
1421                         vim_interface_id: #interface/port VIM id
1422                         ip_address:       #null, or text with IPv4, IPv6 address
1423                         compute_node:     #identification of compute node where PF,VF interface is allocated
1424                         pci:              #PCI address of the NIC that hosts the PF,VF
1425                         vlan:             #physical VLAN used for VF
1426         '''
1427         vm_dict={}
1428         self.logger.debug("refresh_vms status: Getting tenant VM instance information from VIM")
1429         for vm_id in vm_list:
1430             vm={}
1431             try:
1432                 vm_vim = self.get_vminstance(vm_id)
1433                 if vm_vim['status'] in vmStatus2manoFormat:
1434                     vm['status']    =  vmStatus2manoFormat[ vm_vim['status'] ]
1435                 else:
1436                     vm['status']    = "OTHER"
1437                     vm['error_msg'] = "VIM status reported " + vm_vim['status']
1438                 try:
1439                     vm['vim_info']  = yaml.safe_dump(vm_vim, default_flow_style=True, width=256)
1440                 except yaml.representer.RepresenterError:
1441                     vm['vim_info'] = str(vm_vim)
1442                 vm["interfaces"] = []
1443                 if vm_vim.get('fault'):
1444                     vm['error_msg'] = str(vm_vim['fault'])
1445                 #get interfaces
1446                 try:
1447                     self._reload_connection()
1448                     port_dict=self.neutron.list_ports(device_id=vm_id)
1449                     for port in port_dict["ports"]:
1450                         interface={}
1451                         try:
1452                             interface['vim_info'] = yaml.safe_dump(port, default_flow_style=True, width=256)
1453                         except yaml.representer.RepresenterError:
1454                             interface['vim_info'] = str(port)
1455                         interface["mac_address"] = port.get("mac_address")
1456                         interface["vim_net_id"] = port["network_id"]
1457                         interface["vim_interface_id"] = port["id"]
1458                         # check if OS-EXT-SRV-ATTR:host is there,
1459                         # in case of non-admin credentials, it will be missing
1460                         if vm_vim.get('OS-EXT-SRV-ATTR:host'):
1461                             interface["compute_node"] = vm_vim['OS-EXT-SRV-ATTR:host']
1462                         interface["pci"] = None
1463
1464                         # check if binding:profile is there,
1465                         # in case of non-admin credentials, it will be missing
1466                         if port.get('binding:profile'):
1467                             if port['binding:profile'].get('pci_slot'):
1468                                 # TODO: At the moment sr-iov pci addresses are converted to PF pci addresses by setting the slot to 0x00
1469                                 # TODO: This is just a workaround valid for niantinc. Find a better way to do so
1470                                 #   CHANGE DDDD:BB:SS.F to DDDD:BB:00.(F%2)   assuming there are 2 ports per nic
1471                                 pci = port['binding:profile']['pci_slot']
1472                                 # interface["pci"] = pci[:-4] + "00." + str(int(pci[-1]) % 2)
1473                                 interface["pci"] = pci
1474                         interface["vlan"] = None
1475                         #if network is of type vlan and port is of type direct (sr-iov) then set vlan id
1476                         network = self.neutron.show_network(port["network_id"])
1477                         if network['network'].get('provider:network_type') == 'vlan' and \
1478                             port.get("binding:vnic_type") == "direct":
1479                             interface["vlan"] = network['network'].get('provider:segmentation_id')
1480                         ips=[]
1481                         #look for floating ip address
1482                         floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
1483                         if floating_ip_dict.get("floatingips"):
1484                             ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address") )
1485
1486                         for subnet in port["fixed_ips"]:
1487                             ips.append(subnet["ip_address"])
1488                         interface["ip_address"] = ";".join(ips)
1489                         vm["interfaces"].append(interface)
1490                 except Exception as e:
1491                     self.logger.error("Error getting vm interface information " + type(e).__name__ + ": "+  str(e))
1492             except vimconn.vimconnNotFoundException as e:
1493                 self.logger.error("Exception getting vm status: %s", str(e))
1494                 vm['status'] = "DELETED"
1495                 vm['error_msg'] = str(e)
1496             except vimconn.vimconnException as e:
1497                 self.logger.error("Exception getting vm status: %s", str(e))
1498                 vm['status'] = "VIM_ERROR"
1499                 vm['error_msg'] = str(e)
1500             vm_dict[vm_id] = vm
1501         return vm_dict
1502
1503     def action_vminstance(self, vm_id, action_dict):
1504         '''Send and action over a VM instance from VIM
1505         Returns the vm_id if the action was successfully sent to the VIM'''
1506         self.logger.debug("Action over VM '%s': %s", vm_id, str(action_dict))
1507         try:
1508             self._reload_connection()
1509             server = self.nova.servers.find(id=vm_id)
1510             if "start" in action_dict:
1511                 if action_dict["start"]=="rebuild":
1512                     server.rebuild()
1513                 else:
1514                     if server.status=="PAUSED":
1515                         server.unpause()
1516                     elif server.status=="SUSPENDED":
1517                         server.resume()
1518                     elif server.status=="SHUTOFF":
1519                         server.start()
1520             elif "pause" in action_dict:
1521                 server.pause()
1522             elif "resume" in action_dict:
1523                 server.resume()
1524             elif "shutoff" in action_dict or "shutdown" in action_dict:
1525                 server.stop()
1526             elif "forceOff" in action_dict:
1527                 server.stop() #TODO
1528             elif "terminate" in action_dict:
1529                 server.delete()
1530             elif "createImage" in action_dict:
1531                 server.create_image()
1532                 #"path":path_schema,
1533                 #"description":description_schema,
1534                 #"name":name_schema,
1535                 #"metadata":metadata_schema,
1536                 #"imageRef": id_schema,
1537                 #"disk": {"oneOf":[{"type": "null"}, {"type":"string"}] },
1538             elif "rebuild" in action_dict:
1539                 server.rebuild(server.image['id'])
1540             elif "reboot" in action_dict:
1541                 server.reboot() #reboot_type='SOFT'
1542             elif "console" in action_dict:
1543                 console_type = action_dict["console"]
1544                 if console_type == None or console_type == "novnc":
1545                     console_dict = server.get_vnc_console("novnc")
1546                 elif console_type == "xvpvnc":
1547                     console_dict = server.get_vnc_console(console_type)
1548                 elif console_type == "rdp-html5":
1549                     console_dict = server.get_rdp_console(console_type)
1550                 elif console_type == "spice-html5":
1551                     console_dict = server.get_spice_console(console_type)
1552                 else:
1553                     raise vimconn.vimconnException("console type '{}' not allowed".format(console_type),
1554                                                    http_code=vimconn.HTTP_Bad_Request)
1555                 try:
1556                     console_url = console_dict["console"]["url"]
1557                     #parse console_url
1558                     protocol_index = console_url.find("//")
1559                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1560                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1561                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1562                         raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1563                     console_dict2={"protocol": console_url[0:protocol_index],
1564                                   "server":   console_url[protocol_index+2 : port_index],
1565                                   "port":     int(console_url[port_index+1 : suffix_index]),
1566                                   "suffix":   console_url[suffix_index+1:]
1567                                   }
1568                     return console_dict2
1569                 except Exception as e:
1570                     raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1571
1572             return vm_id
1573         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1574             self._format_exception(e)
1575         #TODO insert exception vimconn.HTTP_Unauthorized
1576
1577     ####### VIO Specific Changes #########
1578     def _genrate_vlanID(self):
1579         """
1580          Method to get unused vlanID
1581             Args:
1582                 None
1583             Returns:
1584                 vlanID
1585         """
1586         #Get used VLAN IDs
1587         usedVlanIDs = []
1588         networks = self.get_network_list()
1589         for net in networks:
1590             if net.get('provider:segmentation_id'):
1591                 usedVlanIDs.append(net.get('provider:segmentation_id'))
1592         used_vlanIDs = set(usedVlanIDs)
1593
1594         #find unused VLAN ID
1595         for vlanID_range in self.config.get('dataplane_net_vlan_range'):
1596             try:
1597                 start_vlanid , end_vlanid = map(int, vlanID_range.replace(" ", "").split("-"))
1598                 for vlanID in xrange(start_vlanid, end_vlanid + 1):
1599                     if vlanID not in used_vlanIDs:
1600                         return vlanID
1601             except Exception as exp:
1602                 raise vimconn.vimconnException("Exception {} occurred while generating VLAN ID.".format(exp))
1603         else:
1604             raise vimconn.vimconnConflictException("Unable to create the SRIOV VLAN network."\
1605                 " All given Vlan IDs {} are in use.".format(self.config.get('dataplane_net_vlan_range')))
1606
1607
1608     def _validate_vlan_ranges(self, dataplane_net_vlan_range):
1609         """
1610         Method to validate user given vlanID ranges
1611             Args:  None
1612             Returns: None
1613         """
1614         for vlanID_range in dataplane_net_vlan_range:
1615             vlan_range = vlanID_range.replace(" ", "")
1616             #validate format
1617             vlanID_pattern = r'(\d)*-(\d)*$'
1618             match_obj = re.match(vlanID_pattern, vlan_range)
1619             if not match_obj:
1620                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}.You must provide "\
1621                 "'dataplane_net_vlan_range' in format [start_ID - end_ID].".format(vlanID_range))
1622
1623             start_vlanid , end_vlanid = map(int,vlan_range.split("-"))
1624             if start_vlanid <= 0 :
1625                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1626                 "Start ID can not be zero. For VLAN "\
1627                 "networks valid IDs are 1 to 4094 ".format(vlanID_range))
1628             if end_vlanid > 4094 :
1629                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1630                 "End VLAN ID can not be greater than 4094. For VLAN "\
1631                 "networks valid IDs are 1 to 4094 ".format(vlanID_range))
1632
1633             if start_vlanid > end_vlanid:
1634                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1635                     "You must provide a 'dataplane_net_vlan_range' in format start_ID - end_ID and "\
1636                     "start_ID < end_ID ".format(vlanID_range))
1637
1638 #NOT USED FUNCTIONS
1639
1640     def new_external_port(self, port_data):
1641         #TODO openstack if needed
1642         '''Adds a external port to VIM'''
1643         '''Returns the port identifier'''
1644         return -vimconn.HTTP_Internal_Server_Error, "osconnector.new_external_port() not implemented"
1645
1646     def connect_port_network(self, port_id, network_id, admin=False):
1647         #TODO openstack if needed
1648         '''Connects a external port to a network'''
1649         '''Returns status code of the VIM response'''
1650         return -vimconn.HTTP_Internal_Server_Error, "osconnector.connect_port_network() not implemented"
1651
1652     def new_user(self, user_name, user_passwd, tenant_id=None):
1653         '''Adds a new user to openstack VIM'''
1654         '''Returns the user identifier'''
1655         self.logger.debug("osconnector: Adding a new user to VIM")
1656         try:
1657             self._reload_connection()
1658             user=self.keystone.users.create(user_name, user_passwd, tenant_id=tenant_id)
1659             #self.keystone.tenants.add_user(self.k_creds["username"], #role)
1660             return user.id
1661         except ksExceptions.ConnectionError as e:
1662             error_value=-vimconn.HTTP_Bad_Request
1663             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1664         except ksExceptions.ClientException as e: #TODO remove
1665             error_value=-vimconn.HTTP_Bad_Request
1666             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1667         #TODO insert exception vimconn.HTTP_Unauthorized
1668         #if reaching here is because an exception
1669         if self.debug:
1670             self.logger.debug("new_user " + error_text)
1671         return error_value, error_text
1672
1673     def delete_user(self, user_id):
1674         '''Delete a user from openstack VIM'''
1675         '''Returns the user identifier'''
1676         if self.debug:
1677             print("osconnector: Deleting  a  user from VIM")
1678         try:
1679             self._reload_connection()
1680             self.keystone.users.delete(user_id)
1681             return 1, user_id
1682         except ksExceptions.ConnectionError as e:
1683             error_value=-vimconn.HTTP_Bad_Request
1684             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1685         except ksExceptions.NotFound as e:
1686             error_value=-vimconn.HTTP_Not_Found
1687             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1688         except ksExceptions.ClientException as e: #TODO remove
1689             error_value=-vimconn.HTTP_Bad_Request
1690             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1691         #TODO insert exception vimconn.HTTP_Unauthorized
1692         #if reaching here is because an exception
1693         if self.debug:
1694             print("delete_tenant " + error_text)
1695         return error_value, error_text
1696
1697     def get_hosts_info(self):
1698         '''Get the information of deployed hosts
1699         Returns the hosts content'''
1700         if self.debug:
1701             print("osconnector: Getting Host info from VIM")
1702         try:
1703             h_list=[]
1704             self._reload_connection()
1705             hypervisors = self.nova.hypervisors.list()
1706             for hype in hypervisors:
1707                 h_list.append( hype.to_dict() )
1708             return 1, {"hosts":h_list}
1709         except nvExceptions.NotFound as e:
1710             error_value=-vimconn.HTTP_Not_Found
1711             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1712         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1713             error_value=-vimconn.HTTP_Bad_Request
1714             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1715         #TODO insert exception vimconn.HTTP_Unauthorized
1716         #if reaching here is because an exception
1717         if self.debug:
1718             print("get_hosts_info " + error_text)
1719         return error_value, error_text
1720
1721     def get_hosts(self, vim_tenant):
1722         '''Get the hosts and deployed instances
1723         Returns the hosts content'''
1724         r, hype_dict = self.get_hosts_info()
1725         if r<0:
1726             return r, hype_dict
1727         hypervisors = hype_dict["hosts"]
1728         try:
1729             servers = self.nova.servers.list()
1730             for hype in hypervisors:
1731                 for server in servers:
1732                     if server.to_dict()['OS-EXT-SRV-ATTR:hypervisor_hostname']==hype['hypervisor_hostname']:
1733                         if 'vm' in hype:
1734                             hype['vm'].append(server.id)
1735                         else:
1736                             hype['vm'] = [server.id]
1737             return 1, hype_dict
1738         except nvExceptions.NotFound as e:
1739             error_value=-vimconn.HTTP_Not_Found
1740             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1741         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1742             error_value=-vimconn.HTTP_Bad_Request
1743             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1744         #TODO insert exception vimconn.HTTP_Unauthorized
1745         #if reaching here is because an exception
1746         if self.debug:
1747             print("get_hosts " + error_text)
1748         return error_value, error_text
1749
1750     def new_classification(self, name, ctype, definition):
1751         self.logger.debug(
1752             'Adding a new (Traffic) Classification to VIM, named %s', name)
1753         try:
1754             new_class = None
1755             self._reload_connection()
1756             if ctype not in supportedClassificationTypes:
1757                 raise vimconn.vimconnNotSupportedException(
1758                         'OpenStack VIM connector doesn\'t support provided '
1759                         'Classification Type {}, supported ones are: '
1760                         '{}'.format(ctype, supportedClassificationTypes))
1761             if not self._validate_classification(ctype, definition):
1762                 raise vimconn.vimconnException(
1763                     'Incorrect Classification definition '
1764                     'for the type specified.')
1765             classification_dict = definition
1766             classification_dict['name'] = name
1767
1768             new_class = self.neutron.create_flow_classifier(
1769                 {'flow_classifier': classification_dict})
1770             return new_class['flow_classifier']['id']
1771         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1772                 neExceptions.NeutronException, ConnectionError) as e:
1773             self.logger.error(
1774                 'Creation of Classification failed.')
1775             self._format_exception(e)
1776
1777     def get_classification(self, class_id):
1778         self.logger.debug(" Getting Classification %s from VIM", class_id)
1779         filter_dict = {"id": class_id}
1780         class_list = self.get_classification_list(filter_dict)
1781         if len(class_list) == 0:
1782             raise vimconn.vimconnNotFoundException(
1783                 "Classification '{}' not found".format(class_id))
1784         elif len(class_list) > 1:
1785             raise vimconn.vimconnConflictException(
1786                 "Found more than one Classification with this criteria")
1787         classification = class_list[0]
1788         return classification
1789
1790     def get_classification_list(self, filter_dict={}):
1791         self.logger.debug("Getting Classifications from VIM filter: '%s'",
1792                           str(filter_dict))
1793         try:
1794             self._reload_connection()
1795             if self.api_version3 and "tenant_id" in filter_dict:
1796                 filter_dict['project_id'] = filter_dict.pop('tenant_id')
1797             classification_dict = self.neutron.list_flow_classifier(
1798                 **filter_dict)
1799             classification_list = classification_dict["flow_classifiers"]
1800             self.__classification_os2mano(classification_list)
1801             return classification_list
1802         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1803                 neExceptions.NeutronException, ConnectionError) as e:
1804             self._format_exception(e)
1805
1806     def delete_classification(self, class_id):
1807         self.logger.debug("Deleting Classification '%s' from VIM", class_id)
1808         try:
1809             self._reload_connection()
1810             self.neutron.delete_flow_classifier(class_id)
1811             return class_id
1812         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1813                 ksExceptions.ClientException, neExceptions.NeutronException,
1814                 ConnectionError) as e:
1815             self._format_exception(e)
1816
1817     def new_sfi(self, name, ingress_ports, egress_ports, sfc_encap=True):
1818         self.logger.debug(
1819             "Adding a new Service Function Instance to VIM, named '%s'", name)
1820         try:
1821             new_sfi = None
1822             self._reload_connection()
1823             correlation = None
1824             if sfc_encap:
1825                 # TODO(igordc): must be changed to NSH in Queens
1826                 # (MPLS is a workaround)
1827                 correlation = 'mpls'
1828             if len(ingress_ports) != 1:
1829                 raise vimconn.vimconnNotSupportedException(
1830                     "OpenStack VIM connector can only have "
1831                     "1 ingress port per SFI")
1832             if len(egress_ports) != 1:
1833                 raise vimconn.vimconnNotSupportedException(
1834                     "OpenStack VIM connector can only have "
1835                     "1 egress port per SFI")
1836             sfi_dict = {'name': name,
1837                         'ingress': ingress_ports[0],
1838                         'egress': egress_ports[0],
1839                         'service_function_parameters': {
1840                             'correlation': correlation}}
1841             new_sfi = self.neutron.create_port_pair({'port_pair': sfi_dict})
1842             return new_sfi['port_pair']['id']
1843         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1844                 neExceptions.NeutronException, ConnectionError) as e:
1845             if new_sfi:
1846                 try:
1847                     self.neutron.delete_port_pair_group(
1848                         new_sfi['port_pair']['id'])
1849                 except Exception:
1850                     self.logger.error(
1851                         'Creation of Service Function Instance failed, with '
1852                         'subsequent deletion failure as well.')
1853             self._format_exception(e)
1854
1855     def get_sfi(self, sfi_id):
1856         self.logger.debug(
1857             'Getting Service Function Instance %s from VIM', sfi_id)
1858         filter_dict = {"id": sfi_id}
1859         sfi_list = self.get_sfi_list(filter_dict)
1860         if len(sfi_list) == 0:
1861             raise vimconn.vimconnNotFoundException(
1862                 "Service Function Instance '{}' not found".format(sfi_id))
1863         elif len(sfi_list) > 1:
1864             raise vimconn.vimconnConflictException(
1865                 'Found more than one Service Function Instance '
1866                 'with this criteria')
1867         sfi = sfi_list[0]
1868         return sfi
1869
1870     def get_sfi_list(self, filter_dict={}):
1871         self.logger.debug("Getting Service Function Instances from "
1872                           "VIM filter: '%s'", str(filter_dict))
1873         try:
1874             self._reload_connection()
1875             if self.api_version3 and "tenant_id" in filter_dict:
1876                 filter_dict['project_id'] = filter_dict.pop('tenant_id')
1877             sfi_dict = self.neutron.list_port_pair(**filter_dict)
1878             sfi_list = sfi_dict["port_pairs"]
1879             self.__sfi_os2mano(sfi_list)
1880             return sfi_list
1881         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1882                 neExceptions.NeutronException, ConnectionError) as e:
1883             self._format_exception(e)
1884
1885     def delete_sfi(self, sfi_id):
1886         self.logger.debug("Deleting Service Function Instance '%s' "
1887                           "from VIM", sfi_id)
1888         try:
1889             self._reload_connection()
1890             self.neutron.delete_port_pair(sfi_id)
1891             return sfi_id
1892         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1893                 ksExceptions.ClientException, neExceptions.NeutronException,
1894                 ConnectionError) as e:
1895             self._format_exception(e)
1896
1897     def new_sf(self, name, sfis, sfc_encap=True):
1898         self.logger.debug("Adding a new Service Function to VIM, "
1899                           "named '%s'", name)
1900         try:
1901             new_sf = None
1902             self._reload_connection()
1903             correlation = None
1904             if sfc_encap:
1905                 # TODO(igordc): must be changed to NSH in Queens
1906                 # (MPLS is a workaround)
1907                 correlation = 'mpls'
1908             for instance in sfis:
1909                 sfi = self.get_sfi(instance)
1910                 if sfi.get('sfc_encap') != correlation:
1911                     raise vimconn.vimconnNotSupportedException(
1912                         "OpenStack VIM connector requires all SFIs of the "
1913                         "same SF to share the same SFC Encapsulation")
1914             sf_dict = {'name': name,
1915                        'port_pairs': sfis}
1916             new_sf = self.neutron.create_port_pair_group({
1917                 'port_pair_group': sf_dict})
1918             return new_sf['port_pair_group']['id']
1919         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1920                 neExceptions.NeutronException, ConnectionError) as e:
1921             if new_sf:
1922                 try:
1923                     self.neutron.delete_port_pair_group(
1924                         new_sf['port_pair_group']['id'])
1925                 except Exception:
1926                     self.logger.error(
1927                         'Creation of Service Function failed, with '
1928                         'subsequent deletion failure as well.')
1929             self._format_exception(e)
1930
1931     def get_sf(self, sf_id):
1932         self.logger.debug("Getting Service Function %s from VIM", sf_id)
1933         filter_dict = {"id": sf_id}
1934         sf_list = self.get_sf_list(filter_dict)
1935         if len(sf_list) == 0:
1936             raise vimconn.vimconnNotFoundException(
1937                 "Service Function '{}' not found".format(sf_id))
1938         elif len(sf_list) > 1:
1939             raise vimconn.vimconnConflictException(
1940                 "Found more than one Service Function with this criteria")
1941         sf = sf_list[0]
1942         return sf
1943
1944     def get_sf_list(self, filter_dict={}):
1945         self.logger.debug("Getting Service Function from VIM filter: '%s'",
1946                           str(filter_dict))
1947         try:
1948             self._reload_connection()
1949             if self.api_version3 and "tenant_id" in filter_dict:
1950                 filter_dict['project_id'] = filter_dict.pop('tenant_id')
1951             sf_dict = self.neutron.list_port_pair_group(**filter_dict)
1952             sf_list = sf_dict["port_pair_groups"]
1953             self.__sf_os2mano(sf_list)
1954             return sf_list
1955         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1956                 neExceptions.NeutronException, ConnectionError) as e:
1957             self._format_exception(e)
1958
1959     def delete_sf(self, sf_id):
1960         self.logger.debug("Deleting Service Function '%s' from VIM", sf_id)
1961         try:
1962             self._reload_connection()
1963             self.neutron.delete_port_pair_group(sf_id)
1964             return sf_id
1965         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1966                 ksExceptions.ClientException, neExceptions.NeutronException,
1967                 ConnectionError) as e:
1968             self._format_exception(e)
1969
1970     def new_sfp(self, name, classifications, sfs, sfc_encap=True, spi=None):
1971         self.logger.debug("Adding a new Service Function Path to VIM, "
1972                           "named '%s'", name)
1973         try:
1974             new_sfp = None
1975             self._reload_connection()
1976             if not sfc_encap:
1977                 raise vimconn.vimconnNotSupportedException(
1978                     "OpenStack VIM connector only supports "
1979                     "SFC-Encapsulated chains")
1980             # TODO(igordc): must be changed to NSH in Queens
1981             # (MPLS is a workaround)
1982             correlation = 'mpls'
1983             sfp_dict = {'name': name,
1984                         'flow_classifiers': classifications,
1985                         'port_pair_groups': sfs,
1986                         'chain_parameters': {'correlation': correlation}}
1987             if spi:
1988                 sfp_dict['chain_id'] = spi
1989             new_sfp = self.neutron.create_port_chain({'port_chain': sfp_dict})
1990             return new_sfp["port_chain"]["id"]
1991         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1992                 neExceptions.NeutronException, ConnectionError) as e:
1993             if new_sfp:
1994                 try:
1995                     self.neutron.delete_port_chain(new_sfp['port_chain']['id'])
1996                 except Exception:
1997                     self.logger.error(
1998                         'Creation of Service Function Path failed, with '
1999                         'subsequent deletion failure as well.')
2000             self._format_exception(e)
2001
2002     def get_sfp(self, sfp_id):
2003         self.logger.debug(" Getting Service Function Path %s from VIM", sfp_id)
2004         filter_dict = {"id": sfp_id}
2005         sfp_list = self.get_sfp_list(filter_dict)
2006         if len(sfp_list) == 0:
2007             raise vimconn.vimconnNotFoundException(
2008                 "Service Function Path '{}' not found".format(sfp_id))
2009         elif len(sfp_list) > 1:
2010             raise vimconn.vimconnConflictException(
2011                 "Found more than one Service Function Path with this criteria")
2012         sfp = sfp_list[0]
2013         return sfp
2014
2015     def get_sfp_list(self, filter_dict={}):
2016         self.logger.debug("Getting Service Function Paths from VIM filter: "
2017                           "'%s'", str(filter_dict))
2018         try:
2019             self._reload_connection()
2020             if self.api_version3 and "tenant_id" in filter_dict:
2021                 filter_dict['project_id'] = filter_dict.pop('tenant_id')
2022             sfp_dict = self.neutron.list_port_chain(**filter_dict)
2023             sfp_list = sfp_dict["port_chains"]
2024             self.__sfp_os2mano(sfp_list)
2025             return sfp_list
2026         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2027                 neExceptions.NeutronException, ConnectionError) as e:
2028             self._format_exception(e)
2029
2030     def delete_sfp(self, sfp_id):
2031         self.logger.debug(
2032             "Deleting Service Function Path '%s' from VIM", sfp_id)
2033         try:
2034             self._reload_connection()
2035             self.neutron.delete_port_chain(sfp_id)
2036             return sfp_id
2037         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
2038                 ksExceptions.ClientException, neExceptions.NeutronException,
2039                 ConnectionError) as e:
2040             self._format_exception(e)