osm_ro/vimconn_openstack.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 osconnector implements all the methods to interact with openstack using the python-neutronclient.
  26
  27 For the VNF forwarding graph, The OpenStack VIM connector calls the
  28 networking-sfc Neutron extension methods, whose resources are mapped
  29 to the VIM connector's SFC resources as follows:
  30 - Classification (OSM) -> Flow Classifier (Neutron)
  31 - Service Function Instance (OSM) -> Port Pair (Neutron)
  32 - Service Function (OSM) -> Port Pair Group (Neutron)
  33 - Service Function Path (OSM) -> Port Chain (Neutron)
  34 '''
  35 __author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes, xFlow Research, Igor D.C., Eduardo Sousa"
  36 __date__  = "$22-sep-2017 23:59:59$"
  37
  38 import vimconn
  39 # import json
  40 import logging
  41 import netaddr
  42 import time
  43 import yaml
  44 import random
  45 import re
  46 import copy
  47 from pprint import pformat
  48 from types import StringTypes
  49
  50 from novaclient import client as nClient, exceptions as nvExceptions
  51 from keystoneauth1.identity import v2, v3
  52 from keystoneauth1 import session
  53 import keystoneclient.exceptions as ksExceptions
  54 import keystoneclient.v3.client as ksClient_v3
  55 import keystoneclient.v2_0.client as ksClient_v2
  56 from glanceclient import client as glClient
  57 import glanceclient.exc as gl1Exceptions
  58 from  cinderclient import client as cClient
  59 from httplib import HTTPException
  60 from neutronclient.neutron import client as neClient
  61 from neutronclient.common import exceptions as neExceptions
  62 from requests.exceptions import ConnectionError
  63
  64
  65 """contain the openstack virtual machine status to openmano status"""
  66 vmStatus2manoFormat={'ACTIVE':'ACTIVE',
  67                      'PAUSED':'PAUSED',
  68                      'SUSPENDED': 'SUSPENDED',
  69                      'SHUTOFF':'INACTIVE',
  70                      'BUILD':'BUILD',
  71                      'ERROR':'ERROR','DELETED':'DELETED'
  72                      }
  73 netStatus2manoFormat={'ACTIVE':'ACTIVE','PAUSED':'PAUSED','INACTIVE':'INACTIVE','BUILD':'BUILD','ERROR':'ERROR','DELETED':'DELETED'
  74                      }
  75
  76 supportedClassificationTypes = ['legacy_flow_classifier']
  77
  78 #global var to have a timeout creating and deleting volumes
  79 volume_timeout = 600
  80 server_timeout = 600
  81
  82
  83 class SafeDumper(yaml.SafeDumper):
  84     def represent_data(self, data):
  85         # Openstack APIs use custom subclasses of dict and YAML safe dumper
  86         # is designed to not handle that (reference issue 142 of pyyaml)
  87         if isinstance(data, dict) and data.__class__ != dict:
  88             # A simple solution is to convert those items back to dicts
  89             data = dict(data.items())
  90
  91         return super(SafeDumper, self).represent_data(data)
  92
  93
  94 class vimconnector(vimconn.vimconnector):
  95     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None,
  96                  log_level=None, config={}, persistent_info={}):
  97         '''using common constructor parameters. In this case
  98         'url' is the keystone authorization url,
  99         'url_admin' is not use
 100         '''
 101         api_version = config.get('APIversion')
 102         if api_version and api_version not in ('v3.3', 'v2.0', '2', '3'):
 103             raise vimconn.vimconnException("Invalid value '{}' for config:APIversion. "
 104                                            "Allowed values are 'v3.3', 'v2.0', '2' or '3'".format(api_version))
 105         vim_type = config.get('vim_type')
 106         if vim_type and vim_type not in ('vio', 'VIO'):
 107             raise vimconn.vimconnException("Invalid value '{}' for config:vim_type."
 108                             "Allowed values are 'vio' or 'VIO'".format(vim_type))
 109
 110         if config.get('dataplane_net_vlan_range') is not None:
 111             #validate vlan ranges provided by user
 112             self._validate_vlan_ranges(config.get('dataplane_net_vlan_range'))
 113
 114         vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level,
 115                                       config)
 116
 117         if self.config.get("insecure") and self.config.get("ca_cert"):
 118             raise vimconn.vimconnException("options insecure and ca_cert are mutually exclusive")
 119         self.verify = True
 120         if self.config.get("insecure"):
 121             self.verify = False
 122         if self.config.get("ca_cert"):
 123             self.verify = self.config.get("ca_cert")
 124
 125         if not url:
 126             raise TypeError('url param can not be NoneType')
 127         self.persistent_info = persistent_info
 128         self.availability_zone = persistent_info.get('availability_zone', None)
 129         self.session = persistent_info.get('session', {'reload_client': True})
 130         self.nova = self.session.get('nova')
 131         self.neutron = self.session.get('neutron')
 132         self.cinder = self.session.get('cinder')
 133         self.glance = self.session.get('glance')
 134         # self.glancev1 = self.session.get('glancev1')
 135         self.keystone = self.session.get('keystone')
 136         self.api_version3 = self.session.get('api_version3')
 137         self.vim_type = self.config.get("vim_type")
 138         if self.vim_type:
 139             self.vim_type = self.vim_type.upper()
 140         if self.config.get("use_internal_endpoint"):
 141             self.endpoint_type = "internalURL"
 142         else:
 143             self.endpoint_type = None
 144
 145         self.logger = logging.getLogger('openmano.vim.openstack')
 146
 147         ####### VIO Specific Changes #########
 148         if self.vim_type == "VIO":
 149             self.logger = logging.getLogger('openmano.vim.vio')
 150
 151         if log_level:
 152             self.logger.setLevel( getattr(logging, log_level))
 153
 154     def __getitem__(self, index):
 155         """Get individuals parameters.
 156         Throw KeyError"""
 157         if index == 'project_domain_id':
 158             return self.config.get("project_domain_id")
 159         elif index == 'user_domain_id':
 160             return self.config.get("user_domain_id")
 161         else:
 162             return vimconn.vimconnector.__getitem__(self, index)
 163
 164     def __setitem__(self, index, value):
 165         """Set individuals parameters and it is marked as dirty so to force connection reload.
 166         Throw KeyError"""
 167         if index == 'project_domain_id':
 168             self.config["project_domain_id"] = value
 169         elif index == 'user_domain_id':
 170                 self.config["user_domain_id"] = value
 171         else:
 172             vimconn.vimconnector.__setitem__(self, index, value)
 173         self.session['reload_client'] = True
 174
 175     def serialize(self, value):
 176         """Serialization of python basic types.
 177
 178         In the case value is not serializable a message will be logged and a
 179         simple representation of the data that cannot be converted back to
 180         python is returned.
 181         """
 182         if isinstance(value, StringTypes):
 183             return value
 184
 185         try:
 186             return yaml.dump(value, Dumper=SafeDumper,
 187                              default_flow_style=True, width=256)
 188         except yaml.representer.RepresenterError:
 189                 self.logger.debug(
 190                     'The following entity cannot be serialized in YAML:'
 191                     '\n\n%s\n\n', pformat(value), exc_info=True)
 192                 return str(value)
 193
 194     def _reload_connection(self):
 195         '''Called before any operation, it check if credentials has changed
 196         Throw keystoneclient.apiclient.exceptions.AuthorizationFailure
 197         '''
 198         #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-)
 199         if self.session['reload_client']:
 200             if self.config.get('APIversion'):
 201                 self.api_version3 = self.config['APIversion'] == 'v3.3' or self.config['APIversion'] == '3'
 202             else:  # get from ending auth_url that end with v3 or with v2.0
 203                 self.api_version3 =  self.url.endswith("/v3") or self.url.endswith("/v3/")
 204             self.session['api_version3'] = self.api_version3
 205             if self.api_version3:
 206                 if self.config.get('project_domain_id') or self.config.get('project_domain_name'):
 207                     project_domain_id_default = None
 208                 else:
 209                     project_domain_id_default = 'default'
 210                 if self.config.get('user_domain_id') or self.config.get('user_domain_name'):
 211                     user_domain_id_default = None
 212                 else:
 213                     user_domain_id_default = 'default'
 214                 auth = v3.Password(auth_url=self.url,
 215                                    username=self.user,
 216                                    password=self.passwd,
 217                                    project_name=self.tenant_name,
 218                                    project_id=self.tenant_id,
 219                                    project_domain_id=self.config.get('project_domain_id', project_domain_id_default),
 220                                    user_domain_id=self.config.get('user_domain_id', user_domain_id_default),
 221                                    project_domain_name=self.config.get('project_domain_name'),
 222                                    user_domain_name=self.config.get('user_domain_name'))
 223             else:
 224                 auth = v2.Password(auth_url=self.url,
 225                                    username=self.user,
 226                                    password=self.passwd,
 227                                    tenant_name=self.tenant_name,
 228                                    tenant_id=self.tenant_id)
 229             sess = session.Session(auth=auth, verify=self.verify)
 230             if self.api_version3:
 231                 self.keystone = ksClient_v3.Client(session=sess, endpoint_type=self.endpoint_type)
 232             else:
 233                 self.keystone = ksClient_v2.Client(session=sess, endpoint_type=self.endpoint_type)
 234             self.session['keystone'] = self.keystone
 235             # In order to enable microversion functionality an explicit microversion must be specified in 'config'.
 236             # This implementation approach is due to the warning message in
 237             # https://developer.openstack.org/api-guide/compute/microversions.html
 238             # where it is stated that microversion backwards compatibility is not guaranteed and clients should
 239             # always require an specific microversion.
 240             # To be able to use 'device role tagging' functionality define 'microversion: 2.32' in datacenter config
 241             version = self.config.get("microversion")
 242             if not version:
 243                 version = "2.1"
 244             self.nova = self.session['nova'] = nClient.Client(str(version), session=sess, endpoint_type=self.endpoint_type)
 245             self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess, endpoint_type=self.endpoint_type)
 246             self.cinder = self.session['cinder'] = cClient.Client(2, session=sess, endpoint_type=self.endpoint_type)
 247             if self.endpoint_type == "internalURL":
 248                 glance_service_id = self.keystone.services.list(name="glance")[0].id
 249                 glance_endpoint = self.keystone.endpoints.list(glance_service_id, interface="internal")[0].url
 250             else:
 251                 glance_endpoint = None
 252             self.glance = self.session['glance'] = glClient.Client(2, session=sess, endpoint=glance_endpoint)
 253             #using version 1 of glance client in new_image()
 254             # self.glancev1 = self.session['glancev1'] = glClient.Client('1', session=sess,
 255             #                                                            endpoint=glance_endpoint)
 256             self.session['reload_client'] = False
 257             self.persistent_info['session'] = self.session
 258             # add availablity zone info inside  self.persistent_info
 259             self._set_availablity_zones()
 260             self.persistent_info['availability_zone'] = self.availability_zone
 261
 262     def __net_os2mano(self, net_list_dict):
 263         '''Transform the net openstack format to mano format
 264         net_list_dict can be a list of dict or a single dict'''
 265         if type(net_list_dict) is dict:
 266             net_list_=(net_list_dict,)
 267         elif type(net_list_dict) is list:
 268             net_list_=net_list_dict
 269         else:
 270             raise TypeError("param net_list_dict must be a list or a dictionary")
 271         for net in net_list_:
 272             if net.get('provider:network_type') == "vlan":
 273                 net['type']='data'
 274             else:
 275                 net['type']='bridge'
 276
 277     def __classification_os2mano(self, class_list_dict):
 278         """Transform the openstack format (Flow Classifier) to mano format
 279         (Classification) class_list_dict can be a list of dict or a single dict
 280         """
 281         if isinstance(class_list_dict, dict):
 282             class_list_ = [class_list_dict]
 283         elif isinstance(class_list_dict, list):
 284             class_list_ = class_list_dict
 285         else:
 286             raise TypeError(
 287                 "param class_list_dict must be a list or a dictionary")
 288         for classification in class_list_:
 289             id = classification.pop('id')
 290             name = classification.pop('name')
 291             description = classification.pop('description')
 292             project_id = classification.pop('project_id')
 293             tenant_id = classification.pop('tenant_id')
 294             original_classification = copy.deepcopy(classification)
 295             classification.clear()
 296             classification['ctype'] = 'legacy_flow_classifier'
 297             classification['definition'] = original_classification
 298             classification['id'] = id
 299             classification['name'] = name
 300             classification['description'] = description
 301             classification['project_id'] = project_id
 302             classification['tenant_id'] = tenant_id
 303
 304     def __sfi_os2mano(self, sfi_list_dict):
 305         """Transform the openstack format (Port Pair) to mano format (SFI)
 306         sfi_list_dict can be a list of dict or a single dict
 307         """
 308         if isinstance(sfi_list_dict, dict):
 309             sfi_list_ = [sfi_list_dict]
 310         elif isinstance(sfi_list_dict, list):
 311             sfi_list_ = sfi_list_dict
 312         else:
 313             raise TypeError(
 314                 "param sfi_list_dict must be a list or a dictionary")
 315         for sfi in sfi_list_:
 316             sfi['ingress_ports'] = []
 317             sfi['egress_ports'] = []
 318             if sfi.get('ingress'):
 319                 sfi['ingress_ports'].append(sfi['ingress'])
 320             if sfi.get('egress'):
 321                 sfi['egress_ports'].append(sfi['egress'])
 322             del sfi['ingress']
 323             del sfi['egress']
 324             params = sfi.get('service_function_parameters')
 325             sfc_encap = False
 326             if params:
 327                 correlation = params.get('correlation')
 328                 if correlation:
 329                     sfc_encap = True
 330             sfi['sfc_encap'] = sfc_encap
 331             del sfi['service_function_parameters']
 332
 333     def __sf_os2mano(self, sf_list_dict):
 334         """Transform the openstack format (Port Pair Group) to mano format (SF)
 335         sf_list_dict can be a list of dict or a single dict
 336         """
 337         if isinstance(sf_list_dict, dict):
 338             sf_list_ = [sf_list_dict]
 339         elif isinstance(sf_list_dict, list):
 340             sf_list_ = sf_list_dict
 341         else:
 342             raise TypeError(
 343                 "param sf_list_dict must be a list or a dictionary")
 344         for sf in sf_list_:
 345             del sf['port_pair_group_parameters']
 346             sf['sfis'] = sf['port_pairs']
 347             del sf['port_pairs']
 348
 349     def __sfp_os2mano(self, sfp_list_dict):
 350         """Transform the openstack format (Port Chain) to mano format (SFP)
 351         sfp_list_dict can be a list of dict or a single dict
 352         """
 353         if isinstance(sfp_list_dict, dict):
 354             sfp_list_ = [sfp_list_dict]
 355         elif isinstance(sfp_list_dict, list):
 356             sfp_list_ = sfp_list_dict
 357         else:
 358             raise TypeError(
 359                 "param sfp_list_dict must be a list or a dictionary")
 360         for sfp in sfp_list_:
 361             params = sfp.pop('chain_parameters')
 362             sfc_encap = False
 363             if params:
 364                 correlation = params.get('correlation')
 365                 if correlation:
 366                     sfc_encap = True
 367             sfp['sfc_encap'] = sfc_encap
 368             sfp['spi'] = sfp.pop('chain_id')
 369             sfp['classifications'] = sfp.pop('flow_classifiers')
 370             sfp['service_functions'] = sfp.pop('port_pair_groups')
 371
 372     # placeholder for now; read TODO note below
 373     def _validate_classification(self, type, definition):
 374         # only legacy_flow_classifier Type is supported at this point
 375         return True
 376         # TODO(igordcard): this method should be an abstract method of an
 377         # abstract Classification class to be implemented by the specific
 378         # Types. Also, abstract vimconnector should call the validation
 379         # method before the implemented VIM connectors are called.
 380
 381     def _format_exception(self, exception):
 382         '''Transform a keystone, nova, neutron  exception into a vimconn exception'''
 383         if isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound, ksExceptions.NotFound, gl1Exceptions.HTTPNotFound)):
 384             raise vimconn.vimconnNotFoundException(type(exception).__name__ + ": " + str(exception))
 385         elif isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError,
 386                                ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed)):
 387             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 388         elif isinstance(exception,  (KeyError, nvExceptions.BadRequest, ksExceptions.BadRequest)):
 389             raise vimconn.vimconnException(type(exception).__name__ + ": " + str(exception))
 390         elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException,
 391                                     neExceptions.NeutronException)):
 392             raise vimconn.vimconnUnexpectedResponse(type(exception).__name__ + ": " + str(exception))
 393         elif isinstance(exception, nvExceptions.Conflict):
 394             raise vimconn.vimconnConflictException(type(exception).__name__ + ": " + str(exception))
 395         elif isinstance(exception, vimconn.vimconnException):
 396             raise exception
 397         else:  # ()
 398             self.logger.error("General Exception " + str(exception), exc_info=True)
 399             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 400
 401     def get_tenant_list(self, filter_dict={}):
 402         '''Obtain tenants of VIM
 403         filter_dict can contain the following keys:
 404             name: filter by tenant name
 405             id: filter by tenant uuid/id
 406             <other VIM specific>
 407         Returns the tenant list of dictionaries: [{'name':'<name>, 'id':'<id>, ...}, ...]
 408         '''
 409         self.logger.debug("Getting tenants from VIM filter: '%s'", str(filter_dict))
 410         try:
 411             self._reload_connection()
 412             if self.api_version3:
 413                 project_class_list = self.keystone.projects.list(name=filter_dict.get("name"))
 414             else:
 415                 project_class_list = self.keystone.tenants.findall(**filter_dict)
 416             project_list=[]
 417             for project in project_class_list:
 418                 if filter_dict.get('id') and filter_dict["id"] != project.id:
 419                     continue
 420                 project_list.append(project.to_dict())
 421             return project_list
 422         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError) as e:
 423             self._format_exception(e)
 424
 425     def new_tenant(self, tenant_name, tenant_description):
 426         '''Adds a new tenant to openstack VIM. Returns the tenant identifier'''
 427         self.logger.debug("Adding a new tenant name: %s", tenant_name)
 428         try:
 429             self._reload_connection()
 430             if self.api_version3:
 431                 project = self.keystone.projects.create(tenant_name, self.config.get("project_domain_id", "default"),
 432                                                         description=tenant_description, is_domain=False)
 433             else:
 434                 project = self.keystone.tenants.create(tenant_name, tenant_description)
 435             return project.id
 436         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ksExceptions.BadRequest, ConnectionError)  as e:
 437             self._format_exception(e)
 438
 439     def delete_tenant(self, tenant_id):
 440         '''Delete a tenant from openstack VIM. Returns the old tenant identifier'''
 441         self.logger.debug("Deleting tenant %s from VIM", tenant_id)
 442         try:
 443             self._reload_connection()
 444             if self.api_version3:
 445                 self.keystone.projects.delete(tenant_id)
 446             else:
 447                 self.keystone.tenants.delete(tenant_id)
 448             return tenant_id
 449         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ksExceptions.NotFound, ConnectionError)  as e:
 450             self._format_exception(e)
 451
 452     def new_network(self,net_name, net_type, ip_profile=None, shared=False, vlan=None):
 453         '''Adds a tenant network to VIM. Returns the network identifier'''
 454         self.logger.debug("Adding a new network to VIM name '%s', type '%s'", net_name, net_type)
 455         #self.logger.debug(">>>>>>>>>>>>>>>>>> IP profile %s", str(ip_profile))
 456         try:
 457             new_net = None
 458             self._reload_connection()
 459             network_dict = {'name': net_name, 'admin_state_up': True}
 460             if net_type=="data" or net_type=="ptp":
 461                 if self.config.get('dataplane_physical_net') == None:
 462                     raise vimconn.vimconnConflictException("You must provide a 'dataplane_physical_net' at config value before creating sriov network")
 463                 network_dict["provider:physical_network"] = self.config['dataplane_physical_net'] #"physnet_sriov" #TODO physical
 464                 network_dict["provider:network_type"]     = "vlan"
 465                 if vlan!=None:
 466                     network_dict["provider:network_type"] = vlan
 467
 468                 ####### VIO Specific Changes #########
 469                 if self.vim_type == "VIO":
 470                     if vlan is not None:
 471                         network_dict["provider:segmentation_id"] = vlan
 472                     else:
 473                         if self.config.get('dataplane_net_vlan_range') is None:
 474                             raise vimconn.vimconnConflictException("You must provide "\
 475                                 "'dataplane_net_vlan_range' in format [start_ID - end_ID]"\
 476                                 "at config value before creating sriov network with vlan tag")
 477
 478                         network_dict["provider:segmentation_id"] = self._genrate_vlanID()
 479
 480             network_dict["shared"]=shared
 481             new_net=self.neutron.create_network({'network':network_dict})
 482             #print new_net
 483             #create subnetwork, even if there is no profile
 484             if not ip_profile:
 485                 ip_profile = {}
 486             if not ip_profile.get('subnet_address'):
 487                 #Fake subnet is required
 488                 subnet_rand = random.randint(0, 255)
 489                 ip_profile['subnet_address'] = "192.168.{}.0/24".format(subnet_rand)
 490             if 'ip_version' not in ip_profile:
 491                 ip_profile['ip_version'] = "IPv4"
 492             subnet = {"name":net_name+"-subnet",
 493                     "network_id": new_net["network"]["id"],
 494                     "ip_version": 4 if ip_profile['ip_version']=="IPv4" else 6,
 495                     "cidr": ip_profile['subnet_address']
 496                     }
 497             # Gateway should be set to None if not needed. Otherwise openstack assigns one by default
 498             if ip_profile.get('gateway_address'):
 499                 subnet['gateway_ip'] = ip_profile['gateway_address']
 500             else:
 501                 subnet['gateway_ip'] = None
 502             if ip_profile.get('dns_address'):
 503                 subnet['dns_nameservers'] = ip_profile['dns_address'].split(";")
 504             if 'dhcp_enabled' in ip_profile:
 505                 subnet['enable_dhcp'] = False if \
 506                     ip_profile['dhcp_enabled']=="false" or ip_profile['dhcp_enabled']==False else True
 507             if ip_profile.get('dhcp_start_address'):
 508                 subnet['allocation_pools'] = []
 509                 subnet['allocation_pools'].append(dict())
 510                 subnet['allocation_pools'][0]['start'] = ip_profile['dhcp_start_address']
 511             if ip_profile.get('dhcp_count'):
 512                 #parts = ip_profile['dhcp_start_address'].split('.')
 513                 #ip_int = (int(parts[0]) << 24) + (int(parts[1]) << 16) + (int(parts[2]) << 8) + int(parts[3])
 514                 ip_int = int(netaddr.IPAddress(ip_profile['dhcp_start_address']))
 515                 ip_int += ip_profile['dhcp_count'] - 1
 516                 ip_str = str(netaddr.IPAddress(ip_int))
 517                 subnet['allocation_pools'][0]['end'] = ip_str
 518             #self.logger.debug(">>>>>>>>>>>>>>>>>> Subnet: %s", str(subnet))
 519             self.neutron.create_subnet({"subnet": subnet} )
 520             return new_net["network"]["id"]
 521         except Exception as e:
 522             if new_net:
 523                 self.neutron.delete_network(new_net['network']['id'])
 524             self._format_exception(e)
 525
 526     def get_network_list(self, filter_dict={}):
 527         '''Obtain tenant networks of VIM
 528         Filter_dict can be:
 529             name: network name
 530             id: network uuid
 531             shared: boolean
 532             tenant_id: tenant
 533             admin_state_up: boolean
 534             status: 'ACTIVE'
 535         Returns the network list of dictionaries
 536         '''
 537         self.logger.debug("Getting network from VIM filter: '%s'", str(filter_dict))
 538         try:
 539             self._reload_connection()
 540             filter_dict_os = filter_dict.copy()
 541             if self.api_version3 and "tenant_id" in filter_dict_os:
 542                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')  #T ODO check
 543             net_dict = self.neutron.list_networks(**filter_dict_os)
 544             net_list = net_dict["networks"]
 545             self.__net_os2mano(net_list)
 546             return net_list
 547         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 548             self._format_exception(e)
 549
 550     def get_network(self, net_id):
 551         '''Obtain details of network from VIM
 552         Returns the network information from a network id'''
 553         self.logger.debug(" Getting tenant network %s from VIM", net_id)
 554         filter_dict={"id": net_id}
 555         net_list = self.get_network_list(filter_dict)
 556         if len(net_list)==0:
 557             raise vimconn.vimconnNotFoundException("Network '{}' not found".format(net_id))
 558         elif len(net_list)>1:
 559             raise vimconn.vimconnConflictException("Found more than one network with this criteria")
 560         net = net_list[0]
 561         subnets=[]
 562         for subnet_id in net.get("subnets", () ):
 563             try:
 564                 subnet = self.neutron.show_subnet(subnet_id)
 565             except Exception as e:
 566                 self.logger.error("osconnector.get_network(): Error getting subnet %s %s" % (net_id, str(e)))
 567                 subnet = {"id": subnet_id, "fault": str(e)}
 568             subnets.append(subnet)
 569         net["subnets"] = subnets
 570         net["encapsulation"] = net.get('provider:network_type')
 571         net["segmentation_id"] = net.get('provider:segmentation_id')
 572         return net
 573
 574     def delete_network(self, net_id):
 575         '''Deletes a tenant network from VIM. Returns the old network identifier'''
 576         self.logger.debug("Deleting network '%s' from VIM", net_id)
 577         try:
 578             self._reload_connection()
 579             #delete VM ports attached to this networks before the network
 580             ports = self.neutron.list_ports(network_id=net_id)
 581             for p in ports['ports']:
 582                 try:
 583                     self.neutron.delete_port(p["id"])
 584                 except Exception as e:
 585                     self.logger.error("Error deleting port %s: %s", p["id"], str(e))
 586             self.neutron.delete_network(net_id)
 587             return net_id
 588         except (neExceptions.ConnectionFailed, neExceptions.NetworkNotFoundClient, neExceptions.NeutronException,
 589                 ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 590             self._format_exception(e)
 591
 592     def refresh_nets_status(self, net_list):
 593         '''Get the status of the networks
 594            Params: the list of network identifiers
 595            Returns a dictionary with:
 596                 net_id:         #VIM id of this network
 597                     status:     #Mandatory. Text with one of:
 598                                 #  DELETED (not found at vim)
 599                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 600                                 #  OTHER (Vim reported other status not understood)
 601                                 #  ERROR (VIM indicates an ERROR status)
 602                                 #  ACTIVE, INACTIVE, DOWN (admin down),
 603                                 #  BUILD (on building process)
 604                                 #
 605                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 606                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 607
 608         '''
 609         net_dict={}
 610         for net_id in net_list:
 611             net = {}
 612             try:
 613                 net_vim = self.get_network(net_id)
 614                 if net_vim['status'] in netStatus2manoFormat:
 615                     net["status"] = netStatus2manoFormat[ net_vim['status'] ]
 616                 else:
 617                     net["status"] = "OTHER"
 618                     net["error_msg"] = "VIM status reported " + net_vim['status']
 619
 620                 if net['status'] == "ACTIVE" and not net_vim['admin_state_up']:
 621                     net['status'] = 'DOWN'
 622
 623                 net['vim_info'] = self.serialize(net_vim)
 624
 625                 if net_vim.get('fault'):  #TODO
 626                     net['error_msg'] = str(net_vim['fault'])
 627             except vimconn.vimconnNotFoundException as e:
 628                 self.logger.error("Exception getting net status: %s", str(e))
 629                 net['status'] = "DELETED"
 630                 net['error_msg'] = str(e)
 631             except vimconn.vimconnException as e:
 632                 self.logger.error("Exception getting net status: %s", str(e))
 633                 net['status'] = "VIM_ERROR"
 634                 net['error_msg'] = str(e)
 635             net_dict[net_id] = net
 636         return net_dict
 637
 638     def get_flavor(self, flavor_id):
 639         '''Obtain flavor details from the  VIM. Returns the flavor dict details'''
 640         self.logger.debug("Getting flavor '%s'", flavor_id)
 641         try:
 642             self._reload_connection()
 643             flavor = self.nova.flavors.find(id=flavor_id)
 644             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 645             return flavor.to_dict()
 646         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 647             self._format_exception(e)
 648
 649     def get_flavor_id_from_data(self, flavor_dict):
 650         """Obtain flavor id that match the flavor description
 651            Returns the flavor_id or raises a vimconnNotFoundException
 652            flavor_dict: contains the required ram, vcpus, disk
 653            If 'use_existing_flavors' is set to True at config, the closer flavor that provides same or more ram, vcpus
 654                 and disk is returned. Otherwise a flavor with exactly same ram, vcpus and disk is returned or a
 655                 vimconnNotFoundException is raised
 656         """
 657         exact_match = False if self.config.get('use_existing_flavors') else True
 658         try:
 659             self._reload_connection()
 660             flavor_candidate_id = None
 661             flavor_candidate_data = (10000, 10000, 10000)
 662             flavor_target = (flavor_dict["ram"], flavor_dict["vcpus"], flavor_dict["disk"])
 663             # numa=None
 664             numas = flavor_dict.get("extended", {}).get("numas")
 665             if numas:
 666                 #TODO
 667                 raise vimconn.vimconnNotFoundException("Flavor with EPA still not implemted")
 668                 # if len(numas) > 1:
 669                 #     raise vimconn.vimconnNotFoundException("Cannot find any flavor with more than one numa")
 670                 # numa=numas[0]
 671                 # numas = extended.get("numas")
 672             for flavor in self.nova.flavors.list():
 673                 epa = flavor.get_keys()
 674                 if epa:
 675                     continue
 676                     # TODO
 677                 flavor_data = (flavor.ram, flavor.vcpus, flavor.disk)
 678                 if flavor_data == flavor_target:
 679                     return flavor.id
 680                 elif not exact_match and flavor_target < flavor_data < flavor_candidate_data:
 681                     flavor_candidate_id = flavor.id
 682                     flavor_candidate_data = flavor_data
 683             if not exact_match and flavor_candidate_id:
 684                 return flavor_candidate_id
 685             raise vimconn.vimconnNotFoundException("Cannot find any flavor matching '{}'".format(str(flavor_dict)))
 686         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 687             self._format_exception(e)
 688
 689     def new_flavor(self, flavor_data, change_name_if_used=True):
 690         '''Adds a tenant flavor to openstack VIM
 691         if change_name_if_used is True, it will change name in case of conflict, because it is not supported name repetition
 692         Returns the flavor identifier
 693         '''
 694         self.logger.debug("Adding flavor '%s'", str(flavor_data))
 695         retry=0
 696         max_retries=3
 697         name_suffix = 0
 698         try:
 699             name=flavor_data['name']
 700             while retry<max_retries:
 701                 retry+=1
 702                 try:
 703                     self._reload_connection()
 704                     if change_name_if_used:
 705                         #get used names
 706                         fl_names=[]
 707                         fl=self.nova.flavors.list()
 708                         for f in fl:
 709                             fl_names.append(f.name)
 710                         while name in fl_names:
 711                             name_suffix += 1
 712                             name = flavor_data['name']+"-" + str(name_suffix)
 713
 714                     ram = flavor_data.get('ram',64)
 715                     vcpus = flavor_data.get('vcpus',1)
 716                     numa_properties=None
 717
 718                     extended = flavor_data.get("extended")
 719                     if extended:
 720                         numas=extended.get("numas")
 721                         if numas:
 722                             numa_nodes = len(numas)
 723                             if numa_nodes > 1:
 724                                 return -1, "Can not add flavor with more than one numa"
 725                             numa_properties = {"hw:numa_nodes":str(numa_nodes)}
 726                             numa_properties["hw:mem_page_size"] = "large"
 727                             numa_properties["hw:cpu_policy"] = "dedicated"
 728                             numa_properties["hw:numa_mempolicy"] = "strict"
 729                             if self.vim_type == "VIO":
 730                                 numa_properties["vmware:extra_config"] = '{"numa.nodeAffinity":"0"}'
 731                                 numa_properties["vmware:latency_sensitivity_level"] = "high"
 732                             for numa in numas:
 733                                 #overwrite ram and vcpus
 734                                 #check if key 'memory' is present in numa else use ram value at flavor
 735                                 if 'memory' in numa:
 736                                     ram = numa['memory']*1024
 737                                 #See for reference: https://specs.openstack.org/openstack/nova-specs/specs/mitaka/implemented/virt-driver-cpu-thread-pinning.html
 738                                 if 'paired-threads' in numa:
 739                                     vcpus = numa['paired-threads']*2
 740                                     #cpu_thread_policy "require" implies that the compute node must have an STM architecture
 741                                     numa_properties["hw:cpu_thread_policy"] = "require"
 742                                     numa_properties["hw:cpu_policy"] = "dedicated"
 743                                 elif 'cores' in numa:
 744                                     vcpus = numa['cores']
 745                                     # cpu_thread_policy "prefer" implies that the host must not have an SMT architecture, or a non-SMT architecture will be emulated
 746                                     numa_properties["hw:cpu_thread_policy"] = "isolate"
 747                                     numa_properties["hw:cpu_policy"] = "dedicated"
 748                                 elif 'threads' in numa:
 749                                     vcpus = numa['threads']
 750                                     # cpu_thread_policy "prefer" implies that the host may or may not have an SMT architecture
 751                                     numa_properties["hw:cpu_thread_policy"] = "prefer"
 752                                     numa_properties["hw:cpu_policy"] = "dedicated"
 753                                 # for interface in numa.get("interfaces",() ):
 754                                 #     if interface["dedicated"]=="yes":
 755                                 #         raise vimconn.vimconnException("Passthrough interfaces are not supported for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable)
 756                                 #     #TODO, add the key 'pci_passthrough:alias"="<label at config>:<number ifaces>"' when a way to connect it is available
 757
 758                     #create flavor
 759                     new_flavor=self.nova.flavors.create(name,
 760                                     ram,
 761                                     vcpus,
 762                                     flavor_data.get('disk',0),
 763                                     is_public=flavor_data.get('is_public', True)
 764                                 )
 765                     #add metadata
 766                     if numa_properties:
 767                         new_flavor.set_keys(numa_properties)
 768                     return new_flavor.id
 769                 except nvExceptions.Conflict as e:
 770                     if change_name_if_used and retry < max_retries:
 771                         continue
 772                     self._format_exception(e)
 773         #except nvExceptions.BadRequest as e:
 774         except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError, KeyError) as e:
 775             self._format_exception(e)
 776
 777     def delete_flavor(self,flavor_id):
 778         '''Deletes a tenant flavor from openstack VIM. Returns the old flavor_id
 779         '''
 780         try:
 781             self._reload_connection()
 782             self.nova.flavors.delete(flavor_id)
 783             return flavor_id
 784         #except nvExceptions.BadRequest as e:
 785         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 786             self._format_exception(e)
 787
 788     def new_image(self,image_dict):
 789         '''
 790         Adds a tenant image to VIM. imge_dict is a dictionary with:
 791             name: name
 792             disk_format: qcow2, vhd, vmdk, raw (by default), ...
 793             location: path or URI
 794             public: "yes" or "no"
 795             metadata: metadata of the image
 796         Returns the image_id
 797         '''
 798         retry=0
 799         max_retries=3
 800         while retry<max_retries:
 801             retry+=1
 802             try:
 803                 self._reload_connection()
 804                 #determine format  http://docs.openstack.org/developer/glance/formats.html
 805                 if "disk_format" in image_dict:
 806                     disk_format=image_dict["disk_format"]
 807                 else: #autodiscover based on extension
 808                     if image_dict['location'].endswith(".qcow2"):
 809                         disk_format="qcow2"
 810                     elif image_dict['location'].endswith(".vhd"):
 811                         disk_format="vhd"
 812                     elif image_dict['location'].endswith(".vmdk"):
 813                         disk_format="vmdk"
 814                     elif image_dict['location'].endswith(".vdi"):
 815                         disk_format="vdi"
 816                     elif image_dict['location'].endswith(".iso"):
 817                         disk_format="iso"
 818                     elif image_dict['location'].endswith(".aki"):
 819                         disk_format="aki"
 820                     elif image_dict['location'].endswith(".ari"):
 821                         disk_format="ari"
 822                     elif image_dict['location'].endswith(".ami"):
 823                         disk_format="ami"
 824                     else:
 825                         disk_format="raw"
 826                 self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location'])
 827                 if self.vim_type == "VIO":
 828                     container_format = "bare"
 829                     if 'container_format' in image_dict:
 830                         container_format = image_dict['container_format']
 831                     new_image = self.glance.images.create(name=image_dict['name'], container_format=container_format,
 832                                                           disk_format=disk_format)
 833                 else:
 834                     new_image = self.glance.images.create(name=image_dict['name'])
 835                 if image_dict['location'].startswith("http"):
 836                     # TODO there is not a method to direct download. It must be downloaded locally with requests
 837                     raise vimconn.vimconnNotImplemented("Cannot create image from URL")
 838                 else: #local path
 839                     with open(image_dict['location']) as fimage:
 840                         self.glance.images.upload(new_image.id, fimage)
 841                         #new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 842                         #    container_format="bare", data=fimage, disk_format=disk_format)
 843                 metadata_to_load = image_dict.get('metadata')
 844                 # TODO location is a reserved word for current openstack versions. fixed for VIO please check for openstack
 845                 if self.vim_type == "VIO":
 846                     metadata_to_load['upload_location'] = image_dict['location']
 847                 else:
 848                     metadata_to_load['location'] = image_dict['location']
 849                 self.glance.images.update(new_image.id, **metadata_to_load)
 850                 return new_image.id
 851             except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e:
 852                 self._format_exception(e)
 853             except (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 854                 if retry==max_retries:
 855                     continue
 856                 self._format_exception(e)
 857             except IOError as e:  #can not open the file
 858                 raise vimconn.vimconnConnectionException(type(e).__name__ + ": " + str(e)+ " for " + image_dict['location'],
 859                                                          http_code=vimconn.HTTP_Bad_Request)
 860
 861     def delete_image(self, image_id):
 862         '''Deletes a tenant image from openstack VIM. Returns the old id
 863         '''
 864         try:
 865             self._reload_connection()
 866             self.glance.images.delete(image_id)
 867             return image_id
 868         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, gl1Exceptions.HTTPNotFound, ConnectionError) as e: #TODO remove
 869             self._format_exception(e)
 870
 871     def get_image_id_from_path(self, path):
 872         '''Get the image id from image path in the VIM database. Returns the image_id'''
 873         try:
 874             self._reload_connection()
 875             images = self.glance.images.list()
 876             for image in images:
 877                 if image.metadata.get("location")==path:
 878                     return image.id
 879             raise vimconn.vimconnNotFoundException("image with location '{}' not found".format( path))
 880         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 881             self._format_exception(e)
 882
 883     def get_image_list(self, filter_dict={}):
 884         '''Obtain tenant images from VIM
 885         Filter_dict can be:
 886             id: image id
 887             name: image name
 888             checksum: image checksum
 889         Returns the image list of dictionaries:
 890             [{<the fields at Filter_dict plus some VIM specific>}, ...]
 891             List can be empty
 892         '''
 893         self.logger.debug("Getting image list from VIM filter: '%s'", str(filter_dict))
 894         try:
 895             self._reload_connection()
 896             filter_dict_os = filter_dict.copy()
 897             #First we filter by the available filter fields: name, id. The others are removed.
 898             image_list = self.glance.images.list()
 899             filtered_list = []
 900             for image in image_list:
 901                 try:
 902                     if filter_dict.get("name") and image["name"] != filter_dict["name"]:
 903                         continue
 904                     if filter_dict.get("id") and image["id"] != filter_dict["id"]:
 905                         continue
 906                     if filter_dict.get("checksum") and image["checksum"] != filter_dict["checksum"]:
 907                         continue
 908
 909                     filtered_list.append(image.copy())
 910                 except gl1Exceptions.HTTPNotFound:
 911                     pass
 912             return filtered_list
 913         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 914             self._format_exception(e)
 915
 916     def __wait_for_vm(self, vm_id, status):
 917         """wait until vm is in the desired status and return True.
 918         If the VM gets in ERROR status, return false.
 919         If the timeout is reached generate an exception"""
 920         elapsed_time = 0
 921         while elapsed_time < server_timeout:
 922             vm_status = self.nova.servers.get(vm_id).status
 923             if vm_status == status:
 924                 return True
 925             if vm_status == 'ERROR':
 926                 return False
 927             time.sleep(5)
 928             elapsed_time += 5
 929
 930         # if we exceeded the timeout rollback
 931         if elapsed_time >= server_timeout:
 932             raise vimconn.vimconnException('Timeout waiting for instance ' + vm_id + ' to get ' + status,
 933                                            http_code=vimconn.HTTP_Request_Timeout)
 934
 935     def _get_openstack_availablity_zones(self):
 936         """
 937         Get from openstack availability zones available
 938         :return:
 939         """
 940         try:
 941             openstack_availability_zone = self.nova.availability_zones.list()
 942             openstack_availability_zone = [str(zone.zoneName) for zone in openstack_availability_zone
 943                                            if zone.zoneName != 'internal']
 944             return openstack_availability_zone
 945         except Exception as e:
 946             return None
 947
 948     def _set_availablity_zones(self):
 949         """
 950         Set vim availablity zone
 951         :return:
 952         """
 953
 954         if 'availability_zone' in self.config:
 955             vim_availability_zones = self.config.get('availability_zone')
 956             if isinstance(vim_availability_zones, str):
 957                 self.availability_zone = [vim_availability_zones]
 958             elif isinstance(vim_availability_zones, list):
 959                 self.availability_zone = vim_availability_zones
 960         else:
 961             self.availability_zone = self._get_openstack_availablity_zones()
 962
 963     def _get_vm_availability_zone(self, availability_zone_index, availability_zone_list):
 964         """
 965         Return thge availability zone to be used by the created VM.
 966         :return: The VIM availability zone to be used or None
 967         """
 968         if availability_zone_index is None:
 969             if not self.config.get('availability_zone'):
 970                 return None
 971             elif isinstance(self.config.get('availability_zone'), str):
 972                 return self.config['availability_zone']
 973             else:
 974                 # TODO consider using a different parameter at config for default AV and AV list match
 975                 return self.config['availability_zone'][0]
 976
 977         vim_availability_zones = self.availability_zone
 978         # check if VIM offer enough availability zones describe in the VNFD
 979         if vim_availability_zones and len(availability_zone_list) <= len(vim_availability_zones):
 980             # check if all the names of NFV AV match VIM AV names
 981             match_by_index = False
 982             for av in availability_zone_list:
 983                 if av not in vim_availability_zones:
 984                     match_by_index = True
 985                     break
 986             if match_by_index:
 987                 return vim_availability_zones[availability_zone_index]
 988             else:
 989                 return availability_zone_list[availability_zone_index]
 990         else:
 991             raise vimconn.vimconnConflictException("No enough availability zones at VIM for this deployment")
 992
 993     def new_vminstance(self, name, description, start, image_id, flavor_id, net_list, cloud_config=None, disk_list=None,
 994                        availability_zone_index=None, availability_zone_list=None):
 995         """Adds a VM instance to VIM
 996         Params:
 997             start: indicates if VM must start or boot in pause mode. Ignored
 998             image_id,flavor_id: iamge and flavor uuid
 999             net_list: list of interfaces, each one is a dictionary with:
1000                 name:
1001                 net_id: network uuid to connect
1002                 vpci: virtual vcpi to assign, ignored because openstack lack #TODO
1003                 model: interface model, ignored #TODO
1004                 mac_address: used for  SR-IOV ifaces #TODO for other types
1005                 use: 'data', 'bridge',  'mgmt'
1006                 type: 'virtual', 'PCI-PASSTHROUGH'('PF'), 'SR-IOV'('VF'), 'VFnotShared'
1007                 vim_id: filled/added by this function
1008                 floating_ip: True/False (or it can be None)
1009             'cloud_config': (optional) dictionary with:
1010             'key-pairs': (optional) list of strings with the public key to be inserted to the default user
1011             'users': (optional) list of users to be inserted, each item is a dict with:
1012                 'name': (mandatory) user name,
1013                 'key-pairs': (optional) list of strings with the public key to be inserted to the user
1014             'user-data': (optional) string is a text script to be passed directly to cloud-init
1015             'config-files': (optional). List of files to be transferred. Each item is a dict with:
1016                 'dest': (mandatory) string with the destination absolute path
1017                 'encoding': (optional, by default text). Can be one of:
1018                     'b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64'
1019                 'content' (mandatory): string with the content of the file
1020                 'permissions': (optional) string with file permissions, typically octal notation '0644'
1021                 'owner': (optional) file owner, string with the format 'owner:group'
1022             'boot-data-drive': boolean to indicate if user-data must be passed using a boot drive (hard disk)
1023             'disk_list': (optional) list with additional disks to the VM. Each item is a dict with:
1024                 'image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted
1025                 'size': (mandatory) string with the size of the disk in GB
1026                 'vim_id' (optional) should use this existing volume id
1027             availability_zone_index: Index of availability_zone_list to use for this this VM. None if not AV required
1028             availability_zone_list: list of availability zones given by user in the VNFD descriptor.  Ignore if
1029                 availability_zone_index is None
1030                 #TODO ip, security groups
1031         Returns a tuple with the instance identifier and created_items or raises an exception on error
1032             created_items can be None or a dictionary where this method can include key-values that will be passed to
1033             the method delete_vminstance and action_vminstance. Can be used to store created ports, volumes, etc.
1034             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
1035             as not present.
1036         """
1037         self.logger.debug("new_vminstance input: image='%s' flavor='%s' nics='%s'",image_id, flavor_id,str(net_list))
1038         try:
1039             server = None
1040             created_items = {}
1041             # metadata = {}
1042             net_list_vim = []
1043             external_network = []   # list of external networks to be connected to instance, later on used to create floating_ip
1044             no_secured_ports = []   # List of port-is with port-security disabled
1045             self._reload_connection()
1046             # metadata_vpci = {}   # For a specific neutron plugin
1047             block_device_mapping = None
1048             for net in net_list:
1049                 if not net.get("net_id"):   # skip non connected iface
1050                     continue
1051
1052                 port_dict={
1053                     "network_id": net["net_id"],
1054                     "name": net.get("name"),
1055                     "admin_state_up": True
1056                 }
1057                 if net["type"]=="virtual":
1058                     pass
1059                     # if "vpci" in net:
1060                     #     metadata_vpci[ net["net_id"] ] = [[ net["vpci"], "" ]]
1061                 elif net["type"] == "VF" or net["type"] == "SR-IOV":  # for VF
1062                     # if "vpci" in net:
1063                     #     if "VF" not in metadata_vpci:
1064                     #         metadata_vpci["VF"]=[]
1065                     #     metadata_vpci["VF"].append([ net["vpci"], "" ])
1066                     port_dict["binding:vnic_type"]="direct"
1067                     # VIO specific Changes
1068                     if self.vim_type == "VIO":
1069                         # Need to create port with port_security_enabled = False and no-security-groups
1070                         port_dict["port_security_enabled"]=False
1071                         port_dict["provider_security_groups"]=[]
1072                         port_dict["security_groups"]=[]
1073                 else:   # For PT PCI-PASSTHROUGH
1074                     # VIO specific Changes
1075                     # Current VIO release does not support port with type 'direct-physical'
1076                     # So no need to create virtual port in case of PCI-device.
1077                     # Will update port_dict code when support gets added in next VIO release
1078                     if self.vim_type == "VIO":
1079                         raise vimconn.vimconnNotSupportedException(
1080                             "Current VIO release does not support full passthrough (PT)")
1081                     # if "vpci" in net:
1082                     #     if "PF" not in metadata_vpci:
1083                     #         metadata_vpci["PF"]=[]
1084                     #     metadata_vpci["PF"].append([ net["vpci"], "" ])
1085                     port_dict["binding:vnic_type"]="direct-physical"
1086                 if not port_dict["name"]:
1087                     port_dict["name"]=name
1088                 if net.get("mac_address"):
1089                     port_dict["mac_address"]=net["mac_address"]
1090                 if net.get("ip_address"):
1091                     port_dict["fixed_ips"] = [{'ip_address': net["ip_address"]}]
1092                     # TODO add 'subnet_id': <subnet_id>
1093                 new_port = self.neutron.create_port({"port": port_dict })
1094                 created_items["port:" + str(new_port["port"]["id"])] = True
1095                 net["mac_adress"] = new_port["port"]["mac_address"]
1096                 net["vim_id"] = new_port["port"]["id"]
1097                 # if try to use a network without subnetwork, it will return a emtpy list
1098                 fixed_ips = new_port["port"].get("fixed_ips")
1099                 if fixed_ips:
1100                     net["ip"] = fixed_ips[0].get("ip_address")
1101                 else:
1102                     net["ip"] = None
1103
1104                 port = {"port-id": new_port["port"]["id"]}
1105                 if float(self.nova.api_version.get_string()) >= 2.32:
1106                     port["tag"] = new_port["port"]["name"]
1107                 net_list_vim.append(port)
1108
1109                 if net.get('floating_ip', False):
1110                     net['exit_on_floating_ip_error'] = True
1111                     external_network.append(net)
1112                 elif net['use'] == 'mgmt' and self.config.get('use_floating_ip'):
1113                     net['exit_on_floating_ip_error'] = False
1114                     external_network.append(net)
1115                     net['floating_ip'] = self.config.get('use_floating_ip')
1116
1117                 # If port security is disabled when the port has not yet been attached to the VM, then all vm traffic is dropped.
1118                 # As a workaround we wait until the VM is active and then disable the port-security
1119                 if net.get("port_security") == False and not self.config.get("no_port_security_extension"):
1120                     no_secured_ports.append(new_port["port"]["id"])
1121
1122             # if metadata_vpci:
1123             #     metadata = {"pci_assignement": json.dumps(metadata_vpci)}
1124             #     if len(metadata["pci_assignement"]) >255:
1125             #         #limit the metadata size
1126             #         #metadata["pci_assignement"] = metadata["pci_assignement"][0:255]
1127             #         self.logger.warn("Metadata deleted since it exceeds the expected length (255) ")
1128             #         metadata = {}
1129
1130             self.logger.debug("name '%s' image_id '%s'flavor_id '%s' net_list_vim '%s' description '%s'",
1131                               name, image_id, flavor_id, str(net_list_vim), description)
1132
1133             security_groups = self.config.get('security_groups')
1134             if type(security_groups) is str:
1135                 security_groups = ( security_groups, )
1136             # cloud config
1137             config_drive, userdata = self._create_user_data(cloud_config)
1138
1139             # Create additional volumes in case these are present in disk_list
1140             base_disk_index = ord('b')
1141             if disk_list:
1142                 block_device_mapping = {}
1143                 for disk in disk_list:
1144                     if disk.get('vim_id'):
1145                         block_device_mapping['_vd' + chr(base_disk_index)] = disk['vim_id']
1146                     else:
1147                         if 'image_id' in disk:
1148                             volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1149                                                                 chr(base_disk_index), imageRef=disk['image_id'])
1150                         else:
1151                             volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1152                                                                 chr(base_disk_index))
1153                         created_items["volume:" + str(volume.id)] = True
1154                         block_device_mapping['_vd' + chr(base_disk_index)] = volume.id
1155                     base_disk_index += 1
1156
1157                 # Wait until created volumes are with status available
1158                 elapsed_time = 0
1159                 while elapsed_time < volume_timeout:
1160                     for created_item in created_items:
1161                         v, _, volume_id = created_item.partition(":")
1162                         if v == 'volume':
1163                             if self.cinder.volumes.get(volume_id).status != 'available':
1164                                 break
1165                     else:  # all ready: break from while
1166                         break
1167                     time.sleep(5)
1168                     elapsed_time += 5
1169                 # If we exceeded the timeout rollback
1170                 if elapsed_time >= volume_timeout:
1171                     raise vimconn.vimconnException('Timeout creating volumes for instance ' + name,
1172                                                    http_code=vimconn.HTTP_Request_Timeout)
1173             # get availability Zone
1174             vm_av_zone = self._get_vm_availability_zone(availability_zone_index, availability_zone_list)
1175
1176             self.logger.debug("nova.servers.create({}, {}, {}, nics={}, security_groups={}, "
1177                               "availability_zone={}, key_name={}, userdata={}, config_drive={}, "
1178                               "block_device_mapping={})".format(name, image_id, flavor_id, net_list_vim,
1179                                                                 security_groups, vm_av_zone, self.config.get('keypair'),
1180                                                                 userdata, config_drive, block_device_mapping))
1181             server = self.nova.servers.create(name, image_id, flavor_id, nics=net_list_vim,
1182                                               security_groups=security_groups,
1183                                               availability_zone=vm_av_zone,
1184                                               key_name=self.config.get('keypair'),
1185                                               userdata=userdata,
1186                                               config_drive=config_drive,
1187                                               block_device_mapping=block_device_mapping
1188                                               )  # , description=description)
1189
1190             vm_start_time = time.time()
1191             # Previously mentioned workaround to wait until the VM is active and then disable the port-security
1192             if no_secured_ports:
1193                 self.__wait_for_vm(server.id, 'ACTIVE')
1194
1195             for port_id in no_secured_ports:
1196                 try:
1197                     self.neutron.update_port(port_id,
1198                                              {"port": {"port_security_enabled": False, "security_groups": None}})
1199                 except Exception as e:
1200                     raise vimconn.vimconnException("It was not possible to disable port security for port {}".format(
1201                         port_id))
1202             # print "DONE :-)", server
1203
1204             # pool_id = None
1205             if external_network:
1206                 floating_ips = self.neutron.list_floatingips().get("floatingips", ())
1207             for floating_network in external_network:
1208                 try:
1209                     assigned = False
1210                     while not assigned:
1211                         if floating_ips:
1212                             ip = floating_ips.pop(0)
1213                             if ip.get("port_id", False) or ip.get('tenant_id') != server.tenant_id:
1214                                 continue
1215                             if isinstance(floating_network['floating_ip'], str):
1216                                 if ip.get("floating_network_id") != floating_network['floating_ip']:
1217                                     continue
1218                             free_floating_ip = ip.get("floating_ip_address")
1219                         else:
1220                             if isinstance(floating_network['floating_ip'], str) and \
1221                                 floating_network['floating_ip'].lower() != "true":
1222                                 pool_id = floating_network['floating_ip']
1223                             else:
1224                                 # Find the external network
1225                                 external_nets = list()
1226                                 for net in self.neutron.list_networks()['networks']:
1227                                     if net['router:external']:
1228                                             external_nets.append(net)
1229
1230                                 if len(external_nets) == 0:
1231                                     raise vimconn.vimconnException("Cannot create floating_ip automatically since no external "
1232                                                                    "network is present",
1233                                                                     http_code=vimconn.HTTP_Conflict)
1234                                 if len(external_nets) > 1:
1235                                     raise vimconn.vimconnException("Cannot create floating_ip automatically since multiple "
1236                                                                    "external networks are present",
1237                                                                    http_code=vimconn.HTTP_Conflict)
1238
1239                                 pool_id = external_nets[0].get('id')
1240                             param = {'floatingip': {'floating_network_id': pool_id, 'tenant_id': server.tenant_id}}
1241                             try:
1242                                 # self.logger.debug("Creating floating IP")
1243                                 new_floating_ip = self.neutron.create_floatingip(param)
1244                                 free_floating_ip = new_floating_ip['floatingip']['floating_ip_address']
1245                             except Exception as e:
1246                                 raise vimconn.vimconnException(type(e).__name__ + ": Cannot create new floating_ip " +
1247                                                                str(e), http_code=vimconn.HTTP_Conflict)
1248
1249                         fix_ip = floating_network.get('ip')
1250                         while not assigned:
1251                             try:
1252                                 server.add_floating_ip(free_floating_ip, fix_ip)
1253                                 assigned = True
1254                             except Exception as e:
1255                                 # openstack need some time after VM creation to asign an IP. So retry if fails
1256                                 vm_status = self.nova.servers.get(server.id).status
1257                                 if vm_status != 'ACTIVE' and vm_status != 'ERROR':
1258                                     if time.time() - vm_start_time < server_timeout:
1259                                         time.sleep(5)
1260                                         continue
1261                                 raise vimconn.vimconnException(
1262                                     "Cannot create floating_ip: {} {}".format(type(e).__name__, e),
1263                                     http_code=vimconn.HTTP_Conflict)
1264
1265                 except Exception as e:
1266                     if not floating_network['exit_on_floating_ip_error']:
1267                         self.logger.warn("Cannot create floating_ip. %s", str(e))
1268                         continue
1269                     raise
1270
1271             return server.id, created_items
1272 #        except nvExceptions.NotFound as e:
1273 #            error_value=-vimconn.HTTP_Not_Found
1274 #            error_text= "vm instance %s not found" % vm_id
1275 #        except TypeError as e:
1276 #            raise vimconn.vimconnException(type(e).__name__ + ": "+  str(e), http_code=vimconn.HTTP_Bad_Request)
1277
1278         except Exception as e:
1279             server_id = None
1280             if server:
1281                 server_id = server.id
1282             try:
1283                 self.delete_vminstance(server_id, created_items)
1284             except Exception as e2:
1285                 self.logger.error("new_vminstance rollback fail {}".format(e2))
1286
1287             self._format_exception(e)
1288
1289     def get_vminstance(self,vm_id):
1290         '''Returns the VM instance information from VIM'''
1291         #self.logger.debug("Getting VM from VIM")
1292         try:
1293             self._reload_connection()
1294             server = self.nova.servers.find(id=vm_id)
1295             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
1296             return server.to_dict()
1297         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1298             self._format_exception(e)
1299
1300     def get_vminstance_console(self,vm_id, console_type="vnc"):
1301         '''
1302         Get a console for the virtual machine
1303         Params:
1304             vm_id: uuid of the VM
1305             console_type, can be:
1306                 "novnc" (by default), "xvpvnc" for VNC types,
1307                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
1308         Returns dict with the console parameters:
1309                 protocol: ssh, ftp, http, https, ...
1310                 server:   usually ip address
1311                 port:     the http, ssh, ... port
1312                 suffix:   extra text, e.g. the http path and query string
1313         '''
1314         self.logger.debug("Getting VM CONSOLE from VIM")
1315         try:
1316             self._reload_connection()
1317             server = self.nova.servers.find(id=vm_id)
1318             if console_type == None or console_type == "novnc":
1319                 console_dict = server.get_vnc_console("novnc")
1320             elif console_type == "xvpvnc":
1321                 console_dict = server.get_vnc_console(console_type)
1322             elif console_type == "rdp-html5":
1323                 console_dict = server.get_rdp_console(console_type)
1324             elif console_type == "spice-html5":
1325                 console_dict = server.get_spice_console(console_type)
1326             else:
1327                 raise vimconn.vimconnException("console type '{}' not allowed".format(console_type), http_code=vimconn.HTTP_Bad_Request)
1328
1329             console_dict1 = console_dict.get("console")
1330             if console_dict1:
1331                 console_url = console_dict1.get("url")
1332                 if console_url:
1333                     #parse console_url
1334                     protocol_index = console_url.find("//")
1335                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1336                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1337                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1338                         return -vimconn.HTTP_Internal_Server_Error, "Unexpected response from VIM"
1339                     console_dict={"protocol": console_url[0:protocol_index],
1340                                   "server":   console_url[protocol_index+2:port_index],
1341                                   "port":     console_url[port_index:suffix_index],
1342                                   "suffix":   console_url[suffix_index+1:]
1343                                   }
1344                     protocol_index += 2
1345                     return console_dict
1346             raise vimconn.vimconnUnexpectedResponse("Unexpected response from VIM")
1347
1348         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.BadRequest, ConnectionError) as e:
1349             self._format_exception(e)
1350
1351     def delete_vminstance(self, vm_id, created_items=None):
1352         '''Removes a VM instance from VIM. Returns the old identifier
1353         '''
1354         #print "osconnector: Getting VM from VIM"
1355         if created_items == None:
1356             created_items = {}
1357         try:
1358             self._reload_connection()
1359             # delete VM ports attached to this networks before the virtual machine
1360             for k, v in created_items.items():
1361                 if not v:  # skip already deleted
1362                     continue
1363                 try:
1364                     k_item, _, k_id = k.partition(":")
1365                     if k_item == "port":
1366                         self.neutron.delete_port(k_id)
1367                 except Exception as e:
1368                     self.logger.error("Error deleting port: {}: {}".format(type(e).__name__, e))
1369
1370             # #commented because detaching the volumes makes the servers.delete not work properly ?!?
1371             # #dettach volumes attached
1372             # server = self.nova.servers.get(vm_id)
1373             # volumes_attached_dict = server._info['os-extended-volumes:volumes_attached']   #volume['id']
1374             # #for volume in volumes_attached_dict:
1375             # #    self.cinder.volumes.detach(volume['id'])
1376
1377             if vm_id:
1378                 self.nova.servers.delete(vm_id)
1379
1380             # delete volumes. Although having detached, they should have in active status before deleting
1381             # we ensure in this loop
1382             keep_waiting = True
1383             elapsed_time = 0
1384             while keep_waiting and elapsed_time < volume_timeout:
1385                 keep_waiting = False
1386                 for k, v in created_items.items():
1387                     if not v:  # skip already deleted
1388                         continue
1389                     try:
1390                         k_item, _, k_id = k.partition(":")
1391                         if k_item == "volume":
1392                             if self.cinder.volumes.get(k_id).status != 'available':
1393                                 keep_waiting = True
1394                             else:
1395                                 self.cinder.volumes.delete(k_id)
1396                     except Exception as e:
1397                         self.logger.error("Error deleting volume: {}: {}".format(type(e).__name__, e))
1398                 if keep_waiting:
1399                     time.sleep(1)
1400                     elapsed_time += 1
1401             return None
1402         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
1403             self._format_exception(e)
1404
1405     def refresh_vms_status(self, vm_list):
1406         '''Get the status of the virtual machines and their interfaces/ports
1407            Params: the list of VM identifiers
1408            Returns a dictionary with:
1409                 vm_id:          #VIM id of this Virtual Machine
1410                     status:     #Mandatory. Text with one of:
1411                                 #  DELETED (not found at vim)
1412                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1413                                 #  OTHER (Vim reported other status not understood)
1414                                 #  ERROR (VIM indicates an ERROR status)
1415                                 #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
1416                                 #  CREATING (on building process), ERROR
1417                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
1418                                 #
1419                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
1420                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
1421                     interfaces:
1422                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
1423                         mac_address:      #Text format XX:XX:XX:XX:XX:XX
1424                         vim_net_id:       #network id where this interface is connected
1425                         vim_interface_id: #interface/port VIM id
1426                         ip_address:       #null, or text with IPv4, IPv6 address
1427                         compute_node:     #identification of compute node where PF,VF interface is allocated
1428                         pci:              #PCI address of the NIC that hosts the PF,VF
1429                         vlan:             #physical VLAN used for VF
1430         '''
1431         vm_dict={}
1432         self.logger.debug("refresh_vms status: Getting tenant VM instance information from VIM")
1433         for vm_id in vm_list:
1434             vm={}
1435             try:
1436                 vm_vim = self.get_vminstance(vm_id)
1437                 if vm_vim['status'] in vmStatus2manoFormat:
1438                     vm['status']    =  vmStatus2manoFormat[ vm_vim['status'] ]
1439                 else:
1440                     vm['status']    = "OTHER"
1441                     vm['error_msg'] = "VIM status reported " + vm_vim['status']
1442
1443                 vm['vim_info'] = self.serialize(vm_vim)
1444
1445                 vm["interfaces"] = []
1446                 if vm_vim.get('fault'):
1447                     vm['error_msg'] = str(vm_vim['fault'])
1448                 #get interfaces
1449                 try:
1450                     self._reload_connection()
1451                     port_dict = self.neutron.list_ports(device_id=vm_id)
1452                     for port in port_dict["ports"]:
1453                         interface={}
1454                         interface['vim_info'] = self.serialize(port)
1455                         interface["mac_address"] = port.get("mac_address")
1456                         interface["vim_net_id"] = port["network_id"]
1457                         interface["vim_interface_id"] = port["id"]
1458                         # check if OS-EXT-SRV-ATTR:host is there,
1459                         # in case of non-admin credentials, it will be missing
1460                         if vm_vim.get('OS-EXT-SRV-ATTR:host'):
1461                             interface["compute_node"] = vm_vim['OS-EXT-SRV-ATTR:host']
1462                         interface["pci"] = None
1463
1464                         # check if binding:profile is there,
1465                         # in case of non-admin credentials, it will be missing
1466                         if port.get('binding:profile'):
1467                             if port['binding:profile'].get('pci_slot'):
1468                                 # TODO: At the moment sr-iov pci addresses are converted to PF pci addresses by setting the slot to 0x00
1469                                 # TODO: This is just a workaround valid for niantinc. Find a better way to do so
1470                                 #   CHANGE DDDD:BB:SS.F to DDDD:BB:00.(F%2)   assuming there are 2 ports per nic
1471                                 pci = port['binding:profile']['pci_slot']
1472                                 # interface["pci"] = pci[:-4] + "00." + str(int(pci[-1]) % 2)
1473                                 interface["pci"] = pci
1474                         interface["vlan"] = None
1475                         #if network is of type vlan and port is of type direct (sr-iov) then set vlan id
1476                         network = self.neutron.show_network(port["network_id"])
1477                         if network['network'].get('provider:network_type') == 'vlan' and \
1478                             port.get("binding:vnic_type") == "direct":
1479                             interface["vlan"] = network['network'].get('provider:segmentation_id')
1480                         ips=[]
1481                         #look for floating ip address
1482                         try:
1483                             floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
1484                             if floating_ip_dict.get("floatingips"):
1485                                 ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address") )
1486                         except Exception:
1487                             pass
1488
1489                         for subnet in port["fixed_ips"]:
1490                             ips.append(subnet["ip_address"])
1491                         interface["ip_address"] = ";".join(ips)
1492                         vm["interfaces"].append(interface)
1493                 except Exception as e:
1494                     self.logger.error("Error getting vm interface information {}: {}".format(type(e).__name__, e),
1495                                       exc_info=True)
1496             except vimconn.vimconnNotFoundException as e:
1497                 self.logger.error("Exception getting vm status: %s", str(e))
1498                 vm['status'] = "DELETED"
1499                 vm['error_msg'] = str(e)
1500             except vimconn.vimconnException as e:
1501                 self.logger.error("Exception getting vm status: %s", str(e))
1502                 vm['status'] = "VIM_ERROR"
1503                 vm['error_msg'] = str(e)
1504             vm_dict[vm_id] = vm
1505         return vm_dict
1506
1507     def action_vminstance(self, vm_id, action_dict, created_items={}):
1508         '''Send and action over a VM instance from VIM
1509         Returns None or the console dict if the action was successfully sent to the VIM'''
1510         self.logger.debug("Action over VM '%s': %s", vm_id, str(action_dict))
1511         try:
1512             self._reload_connection()
1513             server = self.nova.servers.find(id=vm_id)
1514             if "start" in action_dict:
1515                 if action_dict["start"]=="rebuild":
1516                     server.rebuild()
1517                 else:
1518                     if server.status=="PAUSED":
1519                         server.unpause()
1520                     elif server.status=="SUSPENDED":
1521                         server.resume()
1522                     elif server.status=="SHUTOFF":
1523                         server.start()
1524             elif "pause" in action_dict:
1525                 server.pause()
1526             elif "resume" in action_dict:
1527                 server.resume()
1528             elif "shutoff" in action_dict or "shutdown" in action_dict:
1529                 server.stop()
1530             elif "forceOff" in action_dict:
1531                 server.stop() #TODO
1532             elif "terminate" in action_dict:
1533                 server.delete()
1534             elif "createImage" in action_dict:
1535                 server.create_image()
1536                 #"path":path_schema,
1537                 #"description":description_schema,
1538                 #"name":name_schema,
1539                 #"metadata":metadata_schema,
1540                 #"imageRef": id_schema,
1541                 #"disk": {"oneOf":[{"type": "null"}, {"type":"string"}] },
1542             elif "rebuild" in action_dict:
1543                 server.rebuild(server.image['id'])
1544             elif "reboot" in action_dict:
1545                 server.reboot() #reboot_type='SOFT'
1546             elif "console" in action_dict:
1547                 console_type = action_dict["console"]
1548                 if console_type == None or console_type == "novnc":
1549                     console_dict = server.get_vnc_console("novnc")
1550                 elif console_type == "xvpvnc":
1551                     console_dict = server.get_vnc_console(console_type)
1552                 elif console_type == "rdp-html5":
1553                     console_dict = server.get_rdp_console(console_type)
1554                 elif console_type == "spice-html5":
1555                     console_dict = server.get_spice_console(console_type)
1556                 else:
1557                     raise vimconn.vimconnException("console type '{}' not allowed".format(console_type),
1558                                                    http_code=vimconn.HTTP_Bad_Request)
1559                 try:
1560                     console_url = console_dict["console"]["url"]
1561                     #parse console_url
1562                     protocol_index = console_url.find("//")
1563                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1564                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1565                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1566                         raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1567                     console_dict2={"protocol": console_url[0:protocol_index],
1568                                   "server":   console_url[protocol_index+2 : port_index],
1569                                   "port":     int(console_url[port_index+1 : suffix_index]),
1570                                   "suffix":   console_url[suffix_index+1:]
1571                                   }
1572                     return console_dict2
1573                 except Exception as e:
1574                     raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1575
1576             return None
1577         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1578             self._format_exception(e)
1579         #TODO insert exception vimconn.HTTP_Unauthorized
1580
1581     ####### VIO Specific Changes #########
1582     def _genrate_vlanID(self):
1583         """
1584          Method to get unused vlanID
1585             Args:
1586                 None
1587             Returns:
1588                 vlanID
1589         """
1590         #Get used VLAN IDs
1591         usedVlanIDs = []
1592         networks = self.get_network_list()
1593         for net in networks:
1594             if net.get('provider:segmentation_id'):
1595                 usedVlanIDs.append(net.get('provider:segmentation_id'))
1596         used_vlanIDs = set(usedVlanIDs)
1597
1598         #find unused VLAN ID
1599         for vlanID_range in self.config.get('dataplane_net_vlan_range'):
1600             try:
1601                 start_vlanid , end_vlanid = map(int, vlanID_range.replace(" ", "").split("-"))
1602                 for vlanID in xrange(start_vlanid, end_vlanid + 1):
1603                     if vlanID not in used_vlanIDs:
1604                         return vlanID
1605             except Exception as exp:
1606                 raise vimconn.vimconnException("Exception {} occurred while generating VLAN ID.".format(exp))
1607         else:
1608             raise vimconn.vimconnConflictException("Unable to create the SRIOV VLAN network."\
1609                 " All given Vlan IDs {} are in use.".format(self.config.get('dataplane_net_vlan_range')))
1610
1611
1612     def _validate_vlan_ranges(self, dataplane_net_vlan_range):
1613         """
1614         Method to validate user given vlanID ranges
1615             Args:  None
1616             Returns: None
1617         """
1618         for vlanID_range in dataplane_net_vlan_range:
1619             vlan_range = vlanID_range.replace(" ", "")
1620             #validate format
1621             vlanID_pattern = r'(\d)*-(\d)*$'
1622             match_obj = re.match(vlanID_pattern, vlan_range)
1623             if not match_obj:
1624                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}.You must provide "\
1625                 "'dataplane_net_vlan_range' in format [start_ID - end_ID].".format(vlanID_range))
1626
1627             start_vlanid , end_vlanid = map(int,vlan_range.split("-"))
1628             if start_vlanid <= 0 :
1629                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1630                 "Start ID can not be zero. For VLAN "\
1631                 "networks valid IDs are 1 to 4094 ".format(vlanID_range))
1632             if end_vlanid > 4094 :
1633                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1634                 "End VLAN ID can not be greater than 4094. For VLAN "\
1635                 "networks valid IDs are 1 to 4094 ".format(vlanID_range))
1636
1637             if start_vlanid > end_vlanid:
1638                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1639                     "You must provide a 'dataplane_net_vlan_range' in format start_ID - end_ID and "\
1640                     "start_ID < end_ID ".format(vlanID_range))
1641
1642 #NOT USED FUNCTIONS
1643
1644     def new_external_port(self, port_data):
1645         #TODO openstack if needed
1646         '''Adds a external port to VIM'''
1647         '''Returns the port identifier'''
1648         return -vimconn.HTTP_Internal_Server_Error, "osconnector.new_external_port() not implemented"
1649
1650     def connect_port_network(self, port_id, network_id, admin=False):
1651         #TODO openstack if needed
1652         '''Connects a external port to a network'''
1653         '''Returns status code of the VIM response'''
1654         return -vimconn.HTTP_Internal_Server_Error, "osconnector.connect_port_network() not implemented"
1655
1656     def new_user(self, user_name, user_passwd, tenant_id=None):
1657         '''Adds a new user to openstack VIM'''
1658         '''Returns the user identifier'''
1659         self.logger.debug("osconnector: Adding a new user to VIM")
1660         try:
1661             self._reload_connection()
1662             user=self.keystone.users.create(user_name, password=user_passwd, default_project=tenant_id)
1663             #self.keystone.tenants.add_user(self.k_creds["username"], #role)
1664             return user.id
1665         except ksExceptions.ConnectionError as e:
1666             error_value=-vimconn.HTTP_Bad_Request
1667             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1668         except ksExceptions.ClientException as e: #TODO remove
1669             error_value=-vimconn.HTTP_Bad_Request
1670             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1671         #TODO insert exception vimconn.HTTP_Unauthorized
1672         #if reaching here is because an exception
1673         self.logger.debug("new_user " + error_text)
1674         return error_value, error_text
1675
1676     def delete_user(self, user_id):
1677         '''Delete a user from openstack VIM'''
1678         '''Returns the user identifier'''
1679         if self.debug:
1680             print("osconnector: Deleting  a  user from VIM")
1681         try:
1682             self._reload_connection()
1683             self.keystone.users.delete(user_id)
1684             return 1, user_id
1685         except ksExceptions.ConnectionError as e:
1686             error_value=-vimconn.HTTP_Bad_Request
1687             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1688         except ksExceptions.NotFound as e:
1689             error_value=-vimconn.HTTP_Not_Found
1690             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1691         except ksExceptions.ClientException as e: #TODO remove
1692             error_value=-vimconn.HTTP_Bad_Request
1693             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1694         #TODO insert exception vimconn.HTTP_Unauthorized
1695         #if reaching here is because an exception
1696             self.logger.debug("delete_tenant " + error_text)
1697         return error_value, error_text
1698
1699     def get_hosts_info(self):
1700         '''Get the information of deployed hosts
1701         Returns the hosts content'''
1702         if self.debug:
1703             print("osconnector: Getting Host info from VIM")
1704         try:
1705             h_list=[]
1706             self._reload_connection()
1707             hypervisors = self.nova.hypervisors.list()
1708             for hype in hypervisors:
1709                 h_list.append( hype.to_dict() )
1710             return 1, {"hosts":h_list}
1711         except nvExceptions.NotFound as e:
1712             error_value=-vimconn.HTTP_Not_Found
1713             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1714         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1715             error_value=-vimconn.HTTP_Bad_Request
1716             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1717         #TODO insert exception vimconn.HTTP_Unauthorized
1718         #if reaching here is because an exception
1719         self.logger.debug("get_hosts_info " + error_text)
1720         return error_value, error_text
1721
1722     def get_hosts(self, vim_tenant):
1723         '''Get the hosts and deployed instances
1724         Returns the hosts content'''
1725         r, hype_dict = self.get_hosts_info()
1726         if r<0:
1727             return r, hype_dict
1728         hypervisors = hype_dict["hosts"]
1729         try:
1730             servers = self.nova.servers.list()
1731             for hype in hypervisors:
1732                 for server in servers:
1733                     if server.to_dict()['OS-EXT-SRV-ATTR:hypervisor_hostname']==hype['hypervisor_hostname']:
1734                         if 'vm' in hype:
1735                             hype['vm'].append(server.id)
1736                         else:
1737                             hype['vm'] = [server.id]
1738             return 1, hype_dict
1739         except nvExceptions.NotFound as e:
1740             error_value=-vimconn.HTTP_Not_Found
1741             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1742         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1743             error_value=-vimconn.HTTP_Bad_Request
1744             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1745         #TODO insert exception vimconn.HTTP_Unauthorized
1746         #if reaching here is because an exception
1747         self.logger.debug("get_hosts " + error_text)
1748         return error_value, error_text
1749
1750     def new_classification(self, name, ctype, definition):
1751         self.logger.debug(
1752             'Adding a new (Traffic) Classification to VIM, named %s', name)
1753         try:
1754             new_class = None
1755             self._reload_connection()
1756             if ctype not in supportedClassificationTypes:
1757                 raise vimconn.vimconnNotSupportedException(
1758                         'OpenStack VIM connector doesn\'t support provided '
1759                         'Classification Type {}, supported ones are: '
1760                         '{}'.format(ctype, supportedClassificationTypes))
1761             if not self._validate_classification(ctype, definition):
1762                 raise vimconn.vimconnException(
1763                     'Incorrect Classification definition '
1764                     'for the type specified.')
1765             classification_dict = definition
1766             classification_dict['name'] = name
1767
1768             new_class = self.neutron.create_sfc_flow_classifier(
1769                 {'flow_classifier': classification_dict})
1770             return new_class['flow_classifier']['id']
1771         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1772                 neExceptions.NeutronException, ConnectionError) as e:
1773             self.logger.error(
1774                 'Creation of Classification failed.')
1775             self._format_exception(e)
1776
1777     def get_classification(self, class_id):
1778         self.logger.debug(" Getting Classification %s from VIM", class_id)
1779         filter_dict = {"id": class_id}
1780         class_list = self.get_classification_list(filter_dict)
1781         if len(class_list) == 0:
1782             raise vimconn.vimconnNotFoundException(
1783                 "Classification '{}' not found".format(class_id))
1784         elif len(class_list) > 1:
1785             raise vimconn.vimconnConflictException(
1786                 "Found more than one Classification with this criteria")
1787         classification = class_list[0]
1788         return classification
1789
1790     def get_classification_list(self, filter_dict={}):
1791         self.logger.debug("Getting Classifications from VIM filter: '%s'",
1792                           str(filter_dict))
1793         try:
1794             filter_dict_os = filter_dict.copy()
1795             self._reload_connection()
1796             if self.api_version3 and "tenant_id" in filter_dict_os:
1797                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1798             classification_dict = self.neutron.list_sfc_flow_classifiers(
1799                 **filter_dict_os)
1800             classification_list = classification_dict["flow_classifiers"]
1801             self.__classification_os2mano(classification_list)
1802             return classification_list
1803         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1804                 neExceptions.NeutronException, ConnectionError) as e:
1805             self._format_exception(e)
1806
1807     def delete_classification(self, class_id):
1808         self.logger.debug("Deleting Classification '%s' from VIM", class_id)
1809         try:
1810             self._reload_connection()
1811             self.neutron.delete_sfc_flow_classifier(class_id)
1812             return class_id
1813         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1814                 ksExceptions.ClientException, neExceptions.NeutronException,
1815                 ConnectionError) as e:
1816             self._format_exception(e)
1817
1818     def new_sfi(self, name, ingress_ports, egress_ports, sfc_encap=True):
1819         self.logger.debug(
1820             "Adding a new Service Function Instance to VIM, named '%s'", name)
1821         try:
1822             new_sfi = None
1823             self._reload_connection()
1824             correlation = None
1825             if sfc_encap:
1826                 correlation = 'nsh'
1827             if len(ingress_ports) != 1:
1828                 raise vimconn.vimconnNotSupportedException(
1829                     "OpenStack VIM connector can only have "
1830                     "1 ingress port per SFI")
1831             if len(egress_ports) != 1:
1832                 raise vimconn.vimconnNotSupportedException(
1833                     "OpenStack VIM connector can only have "
1834                     "1 egress port per SFI")
1835             sfi_dict = {'name': name,
1836                         'ingress': ingress_ports[0],
1837                         'egress': egress_ports[0],
1838                         'service_function_parameters': {
1839                             'correlation': correlation}}
1840             new_sfi = self.neutron.create_sfc_port_pair({'port_pair': sfi_dict})
1841             return new_sfi['port_pair']['id']
1842         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1843                 neExceptions.NeutronException, ConnectionError) as e:
1844             if new_sfi:
1845                 try:
1846                     self.neutron.delete_sfc_port_pair(
1847                         new_sfi['port_pair']['id'])
1848                 except Exception:
1849                     self.logger.error(
1850                         'Creation of Service Function Instance failed, with '
1851                         'subsequent deletion failure as well.')
1852             self._format_exception(e)
1853
1854     def get_sfi(self, sfi_id):
1855         self.logger.debug(
1856             'Getting Service Function Instance %s from VIM', sfi_id)
1857         filter_dict = {"id": sfi_id}
1858         sfi_list = self.get_sfi_list(filter_dict)
1859         if len(sfi_list) == 0:
1860             raise vimconn.vimconnNotFoundException(
1861                 "Service Function Instance '{}' not found".format(sfi_id))
1862         elif len(sfi_list) > 1:
1863             raise vimconn.vimconnConflictException(
1864                 'Found more than one Service Function Instance '
1865                 'with this criteria')
1866         sfi = sfi_list[0]
1867         return sfi
1868
1869     def get_sfi_list(self, filter_dict={}):
1870         self.logger.debug("Getting Service Function Instances from "
1871                           "VIM filter: '%s'", str(filter_dict))
1872         try:
1873             self._reload_connection()
1874             filter_dict_os = filter_dict.copy()
1875             if self.api_version3 and "tenant_id" in filter_dict_os:
1876                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1877             sfi_dict = self.neutron.list_sfc_port_pairs(**filter_dict_os)
1878             sfi_list = sfi_dict["port_pairs"]
1879             self.__sfi_os2mano(sfi_list)
1880             return sfi_list
1881         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1882                 neExceptions.NeutronException, ConnectionError) as e:
1883             self._format_exception(e)
1884
1885     def delete_sfi(self, sfi_id):
1886         self.logger.debug("Deleting Service Function Instance '%s' "
1887                           "from VIM", sfi_id)
1888         try:
1889             self._reload_connection()
1890             self.neutron.delete_sfc_port_pair(sfi_id)
1891             return sfi_id
1892         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1893                 ksExceptions.ClientException, neExceptions.NeutronException,
1894                 ConnectionError) as e:
1895             self._format_exception(e)
1896
1897     def new_sf(self, name, sfis, sfc_encap=True):
1898         self.logger.debug("Adding a new Service Function to VIM, "
1899                           "named '%s'", name)
1900         try:
1901             new_sf = None
1902             self._reload_connection()
1903             # correlation = None
1904             # if sfc_encap:
1905             #     correlation = 'nsh'
1906             for instance in sfis:
1907                 sfi = self.get_sfi(instance)
1908                 if sfi.get('sfc_encap') != sfc_encap:
1909                     raise vimconn.vimconnNotSupportedException(
1910                         "OpenStack VIM connector requires all SFIs of the "
1911                         "same SF to share the same SFC Encapsulation")
1912             sf_dict = {'name': name,
1913                        'port_pairs': sfis}
1914             new_sf = self.neutron.create_sfc_port_pair_group({
1915                 'port_pair_group': sf_dict})
1916             return new_sf['port_pair_group']['id']
1917         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1918                 neExceptions.NeutronException, ConnectionError) as e:
1919             if new_sf:
1920                 try:
1921                     self.neutron.delete_sfc_port_pair_group(
1922                         new_sf['port_pair_group']['id'])
1923                 except Exception:
1924                     self.logger.error(
1925                         'Creation of Service Function failed, with '
1926                         'subsequent deletion failure as well.')
1927             self._format_exception(e)
1928
1929     def get_sf(self, sf_id):
1930         self.logger.debug("Getting Service Function %s from VIM", sf_id)
1931         filter_dict = {"id": sf_id}
1932         sf_list = self.get_sf_list(filter_dict)
1933         if len(sf_list) == 0:
1934             raise vimconn.vimconnNotFoundException(
1935                 "Service Function '{}' not found".format(sf_id))
1936         elif len(sf_list) > 1:
1937             raise vimconn.vimconnConflictException(
1938                 "Found more than one Service Function with this criteria")
1939         sf = sf_list[0]
1940         return sf
1941
1942     def get_sf_list(self, filter_dict={}):
1943         self.logger.debug("Getting Service Function from VIM filter: '%s'",
1944                           str(filter_dict))
1945         try:
1946             self._reload_connection()
1947             filter_dict_os = filter_dict.copy()
1948             if self.api_version3 and "tenant_id" in filter_dict_os:
1949                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1950             sf_dict = self.neutron.list_sfc_port_pair_groups(**filter_dict_os)
1951             sf_list = sf_dict["port_pair_groups"]
1952             self.__sf_os2mano(sf_list)
1953             return sf_list
1954         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1955                 neExceptions.NeutronException, ConnectionError) as e:
1956             self._format_exception(e)
1957
1958     def delete_sf(self, sf_id):
1959         self.logger.debug("Deleting Service Function '%s' from VIM", sf_id)
1960         try:
1961             self._reload_connection()
1962             self.neutron.delete_sfc_port_pair_group(sf_id)
1963             return sf_id
1964         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1965                 ksExceptions.ClientException, neExceptions.NeutronException,
1966                 ConnectionError) as e:
1967             self._format_exception(e)
1968
1969     def new_sfp(self, name, classifications, sfs, sfc_encap=True, spi=None):
1970         self.logger.debug("Adding a new Service Function Path to VIM, "
1971                           "named '%s'", name)
1972         try:
1973             new_sfp = None
1974             self._reload_connection()
1975             # In networking-sfc the MPLS encapsulation is legacy
1976             # should be used when no full SFC Encapsulation is intended
1977             sfc_encap = 'mpls'
1978             if sfc_encap:
1979                 correlation = 'nsh'
1980             sfp_dict = {'name': name,
1981                         'flow_classifiers': classifications,
1982                         'port_pair_groups': sfs,
1983                         'chain_parameters': {'correlation': correlation}}
1984             if spi:
1985                 sfp_dict['chain_id'] = spi
1986             new_sfp = self.neutron.create_sfc_port_chain({'port_chain': sfp_dict})
1987             return new_sfp["port_chain"]["id"]
1988         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1989                 neExceptions.NeutronException, ConnectionError) as e:
1990             if new_sfp:
1991                 try:
1992                     self.neutron.delete_sfc_port_chain(new_sfp['port_chain']['id'])
1993                 except Exception:
1994                     self.logger.error(
1995                         'Creation of Service Function Path failed, with '
1996                         'subsequent deletion failure as well.')
1997             self._format_exception(e)
1998
1999     def get_sfp(self, sfp_id):
2000         self.logger.debug(" Getting Service Function Path %s from VIM", sfp_id)
2001         filter_dict = {"id": sfp_id}
2002         sfp_list = self.get_sfp_list(filter_dict)
2003         if len(sfp_list) == 0:
2004             raise vimconn.vimconnNotFoundException(
2005                 "Service Function Path '{}' not found".format(sfp_id))
2006         elif len(sfp_list) > 1:
2007             raise vimconn.vimconnConflictException(
2008                 "Found more than one Service Function Path with this criteria")
2009         sfp = sfp_list[0]
2010         return sfp
2011
2012     def get_sfp_list(self, filter_dict={}):
2013         self.logger.debug("Getting Service Function Paths from VIM filter: "
2014                           "'%s'", str(filter_dict))
2015         try:
2016             self._reload_connection()
2017             filter_dict_os = filter_dict.copy()
2018             if self.api_version3 and "tenant_id" in filter_dict_os:
2019                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
2020             sfp_dict = self.neutron.list_sfc_port_chains(**filter_dict_os)
2021             sfp_list = sfp_dict["port_chains"]
2022             self.__sfp_os2mano(sfp_list)
2023             return sfp_list
2024         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2025                 neExceptions.NeutronException, ConnectionError) as e:
2026             self._format_exception(e)
2027
2028     def delete_sfp(self, sfp_id):
2029         self.logger.debug(
2030             "Deleting Service Function Path '%s' from VIM", sfp_id)
2031         try:
2032             self._reload_connection()
2033             self.neutron.delete_sfc_port_chain(sfp_id)
2034             return sfp_id
2035         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
2036                 ksExceptions.ClientException, neExceptions.NeutronException,
2037                 ConnectionError) as e:
2038             self._format_exception(e)