osm_ro/vimconn_openstack.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 osconnector implements all the methods to interact with openstack using the python-neutronclient.
  26
  27 For the VNF forwarding graph, The OpenStack VIM connector calls the
  28 networking-sfc Neutron extension methods, whose resources are mapped
  29 to the VIM connector's SFC resources as follows:
  30 - Classification (OSM) -> Flow Classifier (Neutron)
  31 - Service Function Instance (OSM) -> Port Pair (Neutron)
  32 - Service Function (OSM) -> Port Pair Group (Neutron)
  33 - Service Function Path (OSM) -> Port Chain (Neutron)
  34 '''
  35 __author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes, xFlow Research, Igor D.C., Eduardo Sousa"
  36 __date__  = "$22-sep-2017 23:59:59$"
  37
  38 import vimconn
  39 # import json
  40 import logging
  41 import netaddr
  42 import time
  43 import yaml
  44 import random
  45 import re
  46 import copy
  47 from pprint import pformat
  48 from types import StringTypes
  49
  50 from novaclient import client as nClient, exceptions as nvExceptions
  51 from keystoneauth1.identity import v2, v3
  52 from keystoneauth1 import session
  53 import keystoneclient.exceptions as ksExceptions
  54 import keystoneclient.v3.client as ksClient_v3
  55 import keystoneclient.v2_0.client as ksClient_v2
  56 from glanceclient import client as glClient
  57 import glanceclient.exc as gl1Exceptions
  58 from  cinderclient import client as cClient
  59 from httplib import HTTPException
  60 from neutronclient.neutron import client as neClient
  61 from neutronclient.common import exceptions as neExceptions
  62 from requests.exceptions import ConnectionError
  63
  64
  65 """contain the openstack virtual machine status to openmano status"""
  66 vmStatus2manoFormat={'ACTIVE':'ACTIVE',
  67                      'PAUSED':'PAUSED',
  68                      'SUSPENDED': 'SUSPENDED',
  69                      'SHUTOFF':'INACTIVE',
  70                      'BUILD':'BUILD',
  71                      'ERROR':'ERROR','DELETED':'DELETED'
  72                      }
  73 netStatus2manoFormat={'ACTIVE':'ACTIVE','PAUSED':'PAUSED','INACTIVE':'INACTIVE','BUILD':'BUILD','ERROR':'ERROR','DELETED':'DELETED'
  74                      }
  75
  76 supportedClassificationTypes = ['legacy_flow_classifier']
  77
  78 #global var to have a timeout creating and deleting volumes
  79 volume_timeout = 600
  80 server_timeout = 600
  81
  82
  83 class SafeDumper(yaml.SafeDumper):
  84     def represent_data(self, data):
  85         # Openstack APIs use custom subclasses of dict and YAML safe dumper
  86         # is designed to not handle that (reference issue 142 of pyyaml)
  87         if isinstance(data, dict) and data.__class__ != dict:
  88             # A simple solution is to convert those items back to dicts
  89             data = dict(data.items())
  90
  91         return super(SafeDumper, self).represent_data(data)
  92
  93
  94 class vimconnector(vimconn.vimconnector):
  95     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None,
  96                  log_level=None, config={}, persistent_info={}):
  97         '''using common constructor parameters. In this case
  98         'url' is the keystone authorization url,
  99         'url_admin' is not use
 100         '''
 101         api_version = config.get('APIversion')
 102         if api_version and api_version not in ('v3.3', 'v2.0', '2', '3'):
 103             raise vimconn.vimconnException("Invalid value '{}' for config:APIversion. "
 104                                            "Allowed values are 'v3.3', 'v2.0', '2' or '3'".format(api_version))
 105         vim_type = config.get('vim_type')
 106         if vim_type and vim_type not in ('vio', 'VIO'):
 107             raise vimconn.vimconnException("Invalid value '{}' for config:vim_type."
 108                             "Allowed values are 'vio' or 'VIO'".format(vim_type))
 109
 110         if config.get('dataplane_net_vlan_range') is not None:
 111             #validate vlan ranges provided by user
 112             self._validate_vlan_ranges(config.get('dataplane_net_vlan_range'))
 113
 114         vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level,
 115                                       config)
 116
 117         if self.config.get("insecure") and self.config.get("ca_cert"):
 118             raise vimconn.vimconnException("options insecure and ca_cert are mutually exclusive")
 119         self.verify = True
 120         if self.config.get("insecure"):
 121             self.verify = False
 122         if self.config.get("ca_cert"):
 123             self.verify = self.config.get("ca_cert")
 124
 125         if not url:
 126             raise TypeError('url param can not be NoneType')
 127         self.persistent_info = persistent_info
 128         self.availability_zone = persistent_info.get('availability_zone', None)
 129         self.session = persistent_info.get('session', {'reload_client': True})
 130         self.nova = self.session.get('nova')
 131         self.neutron = self.session.get('neutron')
 132         self.cinder = self.session.get('cinder')
 133         self.glance = self.session.get('glance')
 134         # self.glancev1 = self.session.get('glancev1')
 135         self.keystone = self.session.get('keystone')
 136         self.api_version3 = self.session.get('api_version3')
 137         self.vim_type = self.config.get("vim_type")
 138         if self.vim_type:
 139             self.vim_type = self.vim_type.upper()
 140         if self.config.get("use_internal_endpoint"):
 141             self.endpoint_type = "internalURL"
 142         else:
 143             self.endpoint_type = None
 144
 145         self.logger = logging.getLogger('openmano.vim.openstack')
 146
 147         ####### VIO Specific Changes #########
 148         if self.vim_type == "VIO":
 149             self.logger = logging.getLogger('openmano.vim.vio')
 150
 151         if log_level:
 152             self.logger.setLevel( getattr(logging, log_level))
 153
 154     def __getitem__(self, index):
 155         """Get individuals parameters.
 156         Throw KeyError"""
 157         if index == 'project_domain_id':
 158             return self.config.get("project_domain_id")
 159         elif index == 'user_domain_id':
 160             return self.config.get("user_domain_id")
 161         else:
 162             return vimconn.vimconnector.__getitem__(self, index)
 163
 164     def __setitem__(self, index, value):
 165         """Set individuals parameters and it is marked as dirty so to force connection reload.
 166         Throw KeyError"""
 167         if index == 'project_domain_id':
 168             self.config["project_domain_id"] = value
 169         elif index == 'user_domain_id':
 170                 self.config["user_domain_id"] = value
 171         else:
 172             vimconn.vimconnector.__setitem__(self, index, value)
 173         self.session['reload_client'] = True
 174
 175     def serialize(self, value):
 176         """Serialization of python basic types.
 177
 178         In the case value is not serializable a message will be logged and a
 179         simple representation of the data that cannot be converted back to
 180         python is returned.
 181         """
 182         if isinstance(value, StringTypes):
 183             return value
 184
 185         try:
 186             return yaml.dump(value, Dumper=SafeDumper,
 187                              default_flow_style=True, width=256)
 188         except yaml.representer.RepresenterError:
 189                 self.logger.debug(
 190                     'The following entity cannot be serialized in YAML:'
 191                     '\n\n%s\n\n', pformat(value), exc_info=True)
 192                 return str(value)
 193
 194     def _reload_connection(self):
 195         '''Called before any operation, it check if credentials has changed
 196         Throw keystoneclient.apiclient.exceptions.AuthorizationFailure
 197         '''
 198         #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-)
 199         if self.session['reload_client']:
 200             if self.config.get('APIversion'):
 201                 self.api_version3 = self.config['APIversion'] == 'v3.3' or self.config['APIversion'] == '3'
 202             else:  # get from ending auth_url that end with v3 or with v2.0
 203                 self.api_version3 =  self.url.endswith("/v3") or self.url.endswith("/v3/")
 204             self.session['api_version3'] = self.api_version3
 205             if self.api_version3:
 206                 if self.config.get('project_domain_id') or self.config.get('project_domain_name'):
 207                     project_domain_id_default = None
 208                 else:
 209                     project_domain_id_default = 'default'
 210                 if self.config.get('user_domain_id') or self.config.get('user_domain_name'):
 211                     user_domain_id_default = None
 212                 else:
 213                     user_domain_id_default = 'default'
 214                 auth = v3.Password(auth_url=self.url,
 215                                    username=self.user,
 216                                    password=self.passwd,
 217                                    project_name=self.tenant_name,
 218                                    project_id=self.tenant_id,
 219                                    project_domain_id=self.config.get('project_domain_id', project_domain_id_default),
 220                                    user_domain_id=self.config.get('user_domain_id', user_domain_id_default),
 221                                    project_domain_name=self.config.get('project_domain_name'),
 222                                    user_domain_name=self.config.get('user_domain_name'))
 223             else:
 224                 auth = v2.Password(auth_url=self.url,
 225                                    username=self.user,
 226                                    password=self.passwd,
 227                                    tenant_name=self.tenant_name,
 228                                    tenant_id=self.tenant_id)
 229             sess = session.Session(auth=auth, verify=self.verify)
 230             if self.api_version3:
 231                 self.keystone = ksClient_v3.Client(session=sess, endpoint_type=self.endpoint_type)
 232             else:
 233                 self.keystone = ksClient_v2.Client(session=sess, endpoint_type=self.endpoint_type)
 234             self.session['keystone'] = self.keystone
 235             # In order to enable microversion functionality an explicit microversion must be specified in 'config'.
 236             # This implementation approach is due to the warning message in
 237             # https://developer.openstack.org/api-guide/compute/microversions.html
 238             # where it is stated that microversion backwards compatibility is not guaranteed and clients should
 239             # always require an specific microversion.
 240             # To be able to use 'device role tagging' functionality define 'microversion: 2.32' in datacenter config
 241             version = self.config.get("microversion")
 242             if not version:
 243                 version = "2.1"
 244             self.nova = self.session['nova'] = nClient.Client(str(version), session=sess, endpoint_type=self.endpoint_type)
 245             self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess, endpoint_type=self.endpoint_type)
 246             self.cinder = self.session['cinder'] = cClient.Client(2, session=sess, endpoint_type=self.endpoint_type)
 247             if self.endpoint_type == "internalURL":
 248                 glance_service_id = self.keystone.services.list(name="glance")[0].id
 249                 glance_endpoint = self.keystone.endpoints.list(glance_service_id, interface="internal")[0].url
 250             else:
 251                 glance_endpoint = None
 252             self.glance = self.session['glance'] = glClient.Client(2, session=sess, endpoint=glance_endpoint)
 253             #using version 1 of glance client in new_image()
 254             # self.glancev1 = self.session['glancev1'] = glClient.Client('1', session=sess,
 255             #                                                            endpoint=glance_endpoint)
 256             self.session['reload_client'] = False
 257             self.persistent_info['session'] = self.session
 258             # add availablity zone info inside  self.persistent_info
 259             self._set_availablity_zones()
 260             self.persistent_info['availability_zone'] = self.availability_zone
 261
 262     def __net_os2mano(self, net_list_dict):
 263         '''Transform the net openstack format to mano format
 264         net_list_dict can be a list of dict or a single dict'''
 265         if type(net_list_dict) is dict:
 266             net_list_=(net_list_dict,)
 267         elif type(net_list_dict) is list:
 268             net_list_=net_list_dict
 269         else:
 270             raise TypeError("param net_list_dict must be a list or a dictionary")
 271         for net in net_list_:
 272             if net.get('provider:network_type') == "vlan":
 273                 net['type']='data'
 274             else:
 275                 net['type']='bridge'
 276
 277     def __classification_os2mano(self, class_list_dict):
 278         """Transform the openstack format (Flow Classifier) to mano format
 279         (Classification) class_list_dict can be a list of dict or a single dict
 280         """
 281         if isinstance(class_list_dict, dict):
 282             class_list_ = [class_list_dict]
 283         elif isinstance(class_list_dict, list):
 284             class_list_ = class_list_dict
 285         else:
 286             raise TypeError(
 287                 "param class_list_dict must be a list or a dictionary")
 288         for classification in class_list_:
 289             id = classification.pop('id')
 290             name = classification.pop('name')
 291             description = classification.pop('description')
 292             project_id = classification.pop('project_id')
 293             tenant_id = classification.pop('tenant_id')
 294             original_classification = copy.deepcopy(classification)
 295             classification.clear()
 296             classification['ctype'] = 'legacy_flow_classifier'
 297             classification['definition'] = original_classification
 298             classification['id'] = id
 299             classification['name'] = name
 300             classification['description'] = description
 301             classification['project_id'] = project_id
 302             classification['tenant_id'] = tenant_id
 303
 304     def __sfi_os2mano(self, sfi_list_dict):
 305         """Transform the openstack format (Port Pair) to mano format (SFI)
 306         sfi_list_dict can be a list of dict or a single dict
 307         """
 308         if isinstance(sfi_list_dict, dict):
 309             sfi_list_ = [sfi_list_dict]
 310         elif isinstance(sfi_list_dict, list):
 311             sfi_list_ = sfi_list_dict
 312         else:
 313             raise TypeError(
 314                 "param sfi_list_dict must be a list or a dictionary")
 315         for sfi in sfi_list_:
 316             sfi['ingress_ports'] = []
 317             sfi['egress_ports'] = []
 318             if sfi.get('ingress'):
 319                 sfi['ingress_ports'].append(sfi['ingress'])
 320             if sfi.get('egress'):
 321                 sfi['egress_ports'].append(sfi['egress'])
 322             del sfi['ingress']
 323             del sfi['egress']
 324             params = sfi.get('service_function_parameters')
 325             sfc_encap = False
 326             if params:
 327                 correlation = params.get('correlation')
 328                 if correlation:
 329                     sfc_encap = True
 330             sfi['sfc_encap'] = sfc_encap
 331             del sfi['service_function_parameters']
 332
 333     def __sf_os2mano(self, sf_list_dict):
 334         """Transform the openstack format (Port Pair Group) to mano format (SF)
 335         sf_list_dict can be a list of dict or a single dict
 336         """
 337         if isinstance(sf_list_dict, dict):
 338             sf_list_ = [sf_list_dict]
 339         elif isinstance(sf_list_dict, list):
 340             sf_list_ = sf_list_dict
 341         else:
 342             raise TypeError(
 343                 "param sf_list_dict must be a list or a dictionary")
 344         for sf in sf_list_:
 345             del sf['port_pair_group_parameters']
 346             sf['sfis'] = sf['port_pairs']
 347             del sf['port_pairs']
 348
 349     def __sfp_os2mano(self, sfp_list_dict):
 350         """Transform the openstack format (Port Chain) to mano format (SFP)
 351         sfp_list_dict can be a list of dict or a single dict
 352         """
 353         if isinstance(sfp_list_dict, dict):
 354             sfp_list_ = [sfp_list_dict]
 355         elif isinstance(sfp_list_dict, list):
 356             sfp_list_ = sfp_list_dict
 357         else:
 358             raise TypeError(
 359                 "param sfp_list_dict must be a list or a dictionary")
 360         for sfp in sfp_list_:
 361             params = sfp.pop('chain_parameters')
 362             sfc_encap = False
 363             if params:
 364                 correlation = params.get('correlation')
 365                 if correlation:
 366                     sfc_encap = True
 367             sfp['sfc_encap'] = sfc_encap
 368             sfp['spi'] = sfp.pop('chain_id')
 369             sfp['classifications'] = sfp.pop('flow_classifiers')
 370             sfp['service_functions'] = sfp.pop('port_pair_groups')
 371
 372     # placeholder for now; read TODO note below
 373     def _validate_classification(self, type, definition):
 374         # only legacy_flow_classifier Type is supported at this point
 375         return True
 376         # TODO(igordcard): this method should be an abstract method of an
 377         # abstract Classification class to be implemented by the specific
 378         # Types. Also, abstract vimconnector should call the validation
 379         # method before the implemented VIM connectors are called.
 380
 381     def _format_exception(self, exception):
 382         '''Transform a keystone, nova, neutron  exception into a vimconn exception'''
 383         if isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound, ksExceptions.NotFound, gl1Exceptions.HTTPNotFound)):
 384             raise vimconn.vimconnNotFoundException(type(exception).__name__ + ": " + str(exception))
 385         elif isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError,
 386                                ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed)):
 387             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 388         elif isinstance(exception,  (KeyError, nvExceptions.BadRequest, ksExceptions.BadRequest)):
 389             raise vimconn.vimconnException(type(exception).__name__ + ": " + str(exception))
 390         elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException,
 391                                     neExceptions.NeutronException)):
 392             raise vimconn.vimconnUnexpectedResponse(type(exception).__name__ + ": " + str(exception))
 393         elif isinstance(exception, nvExceptions.Conflict):
 394             raise vimconn.vimconnConflictException(type(exception).__name__ + ": " + str(exception))
 395         elif isinstance(exception, vimconn.vimconnException):
 396             raise exception
 397         else:  # ()
 398             self.logger.error("General Exception " + str(exception), exc_info=True)
 399             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 400
 401     def get_tenant_list(self, filter_dict={}):
 402         '''Obtain tenants of VIM
 403         filter_dict can contain the following keys:
 404             name: filter by tenant name
 405             id: filter by tenant uuid/id
 406             <other VIM specific>
 407         Returns the tenant list of dictionaries: [{'name':'<name>, 'id':'<id>, ...}, ...]
 408         '''
 409         self.logger.debug("Getting tenants from VIM filter: '%s'", str(filter_dict))
 410         try:
 411             self._reload_connection()
 412             if self.api_version3:
 413                 project_class_list = self.keystone.projects.list(name=filter_dict.get("name"))
 414             else:
 415                 project_class_list = self.keystone.tenants.findall(**filter_dict)
 416             project_list=[]
 417             for project in project_class_list:
 418                 if filter_dict.get('id') and filter_dict["id"] != project.id:
 419                     continue
 420                 project_list.append(project.to_dict())
 421             return project_list
 422         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError) as e:
 423             self._format_exception(e)
 424
 425     def new_tenant(self, tenant_name, tenant_description):
 426         '''Adds a new tenant to openstack VIM. Returns the tenant identifier'''
 427         self.logger.debug("Adding a new tenant name: %s", tenant_name)
 428         try:
 429             self._reload_connection()
 430             if self.api_version3:
 431                 project = self.keystone.projects.create(tenant_name, self.config.get("project_domain_id", "default"),
 432                                                         description=tenant_description, is_domain=False)
 433             else:
 434                 project = self.keystone.tenants.create(tenant_name, tenant_description)
 435             return project.id
 436         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ksExceptions.BadRequest, ConnectionError)  as e:
 437             self._format_exception(e)
 438
 439     def delete_tenant(self, tenant_id):
 440         '''Delete a tenant from openstack VIM. Returns the old tenant identifier'''
 441         self.logger.debug("Deleting tenant %s from VIM", tenant_id)
 442         try:
 443             self._reload_connection()
 444             if self.api_version3:
 445                 self.keystone.projects.delete(tenant_id)
 446             else:
 447                 self.keystone.tenants.delete(tenant_id)
 448             return tenant_id
 449         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ksExceptions.NotFound, ConnectionError)  as e:
 450             self._format_exception(e)
 451
 452     def new_network(self,net_name, net_type, ip_profile=None, shared=False, vlan=None):
 453         '''Adds a tenant network to VIM. Returns the network identifier'''
 454         self.logger.debug("Adding a new network to VIM name '%s', type '%s'", net_name, net_type)
 455         #self.logger.debug(">>>>>>>>>>>>>>>>>> IP profile %s", str(ip_profile))
 456         try:
 457             new_net = None
 458             self._reload_connection()
 459             network_dict = {'name': net_name, 'admin_state_up': True}
 460             if net_type=="data" or net_type=="ptp":
 461                 if self.config.get('dataplane_physical_net') == None:
 462                     raise vimconn.vimconnConflictException("You must provide a 'dataplane_physical_net' at config value before creating sriov network")
 463                 network_dict["provider:physical_network"] = self.config['dataplane_physical_net'] #"physnet_sriov" #TODO physical
 464                 network_dict["provider:network_type"]     = "vlan"
 465                 if vlan!=None:
 466                     network_dict["provider:network_type"] = vlan
 467
 468                 ####### VIO Specific Changes #########
 469                 if self.vim_type == "VIO":
 470                     if vlan is not None:
 471                         network_dict["provider:segmentation_id"] = vlan
 472                     else:
 473                         if self.config.get('dataplane_net_vlan_range') is None:
 474                             raise vimconn.vimconnConflictException("You must provide "\
 475                                 "'dataplane_net_vlan_range' in format [start_ID - end_ID]"\
 476                                 "at config value before creating sriov network with vlan tag")
 477
 478                         network_dict["provider:segmentation_id"] = self._genrate_vlanID()
 479
 480             network_dict["shared"]=shared
 481             new_net=self.neutron.create_network({'network':network_dict})
 482             #print new_net
 483             #create subnetwork, even if there is no profile
 484             if not ip_profile:
 485                 ip_profile = {}
 486             if not ip_profile.get('subnet_address'):
 487                 #Fake subnet is required
 488                 subnet_rand = random.randint(0, 255)
 489                 ip_profile['subnet_address'] = "192.168.{}.0/24".format(subnet_rand)
 490             if 'ip_version' not in ip_profile:
 491                 ip_profile['ip_version'] = "IPv4"
 492             subnet = {"name":net_name+"-subnet",
 493                     "network_id": new_net["network"]["id"],
 494                     "ip_version": 4 if ip_profile['ip_version']=="IPv4" else 6,
 495                     "cidr": ip_profile['subnet_address']
 496                     }
 497             # Gateway should be set to None if not needed. Otherwise openstack assigns one by default
 498             if ip_profile.get('gateway_address'):
 499                 subnet['gateway_ip'] = ip_profile['gateway_address']
 500             else:
 501                 subnet['gateway_ip'] = None
 502             if ip_profile.get('dns_address'):
 503                 subnet['dns_nameservers'] = ip_profile['dns_address'].split(";")
 504             if 'dhcp_enabled' in ip_profile:
 505                 subnet['enable_dhcp'] = False if \
 506                     ip_profile['dhcp_enabled']=="false" or ip_profile['dhcp_enabled']==False else True
 507             if ip_profile.get('dhcp_start_address'):
 508                 subnet['allocation_pools'] = []
 509                 subnet['allocation_pools'].append(dict())
 510                 subnet['allocation_pools'][0]['start'] = ip_profile['dhcp_start_address']
 511             if ip_profile.get('dhcp_count'):
 512                 #parts = ip_profile['dhcp_start_address'].split('.')
 513                 #ip_int = (int(parts[0]) << 24) + (int(parts[1]) << 16) + (int(parts[2]) << 8) + int(parts[3])
 514                 ip_int = int(netaddr.IPAddress(ip_profile['dhcp_start_address']))
 515                 ip_int += ip_profile['dhcp_count'] - 1
 516                 ip_str = str(netaddr.IPAddress(ip_int))
 517                 subnet['allocation_pools'][0]['end'] = ip_str
 518             #self.logger.debug(">>>>>>>>>>>>>>>>>> Subnet: %s", str(subnet))
 519             self.neutron.create_subnet({"subnet": subnet} )
 520             return new_net["network"]["id"]
 521         except Exception as e:
 522             if new_net:
 523                 self.neutron.delete_network(new_net['network']['id'])
 524             self._format_exception(e)
 525
 526     def get_network_list(self, filter_dict={}):
 527         '''Obtain tenant networks of VIM
 528         Filter_dict can be:
 529             name: network name
 530             id: network uuid
 531             shared: boolean
 532             tenant_id: tenant
 533             admin_state_up: boolean
 534             status: 'ACTIVE'
 535         Returns the network list of dictionaries
 536         '''
 537         self.logger.debug("Getting network from VIM filter: '%s'", str(filter_dict))
 538         try:
 539             self._reload_connection()
 540             filter_dict_os = filter_dict.copy()
 541             if self.api_version3 and "tenant_id" in filter_dict_os:
 542                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')  #T ODO check
 543             net_dict = self.neutron.list_networks(**filter_dict_os)
 544             net_list = net_dict["networks"]
 545             self.__net_os2mano(net_list)
 546             return net_list
 547         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 548             self._format_exception(e)
 549
 550     def get_network(self, net_id):
 551         '''Obtain details of network from VIM
 552         Returns the network information from a network id'''
 553         self.logger.debug(" Getting tenant network %s from VIM", net_id)
 554         filter_dict={"id": net_id}
 555         net_list = self.get_network_list(filter_dict)
 556         if len(net_list)==0:
 557             raise vimconn.vimconnNotFoundException("Network '{}' not found".format(net_id))
 558         elif len(net_list)>1:
 559             raise vimconn.vimconnConflictException("Found more than one network with this criteria")
 560         net = net_list[0]
 561         subnets=[]
 562         for subnet_id in net.get("subnets", () ):
 563             try:
 564                 subnet = self.neutron.show_subnet(subnet_id)
 565             except Exception as e:
 566                 self.logger.error("osconnector.get_network(): Error getting subnet %s %s" % (net_id, str(e)))
 567                 subnet = {"id": subnet_id, "fault": str(e)}
 568             subnets.append(subnet)
 569         net["subnets"] = subnets
 570         net["encapsulation"] = net.get('provider:network_type')
 571         net["encapsulation_type"] = net.get('provider:network_type')
 572         net["segmentation_id"] = net.get('provider:segmentation_id')
 573         net["encapsulation_id"] = net.get('provider:segmentation_id')
 574         return net
 575
 576     def delete_network(self, net_id):
 577         '''Deletes a tenant network from VIM. Returns the old network identifier'''
 578         self.logger.debug("Deleting network '%s' from VIM", net_id)
 579         try:
 580             self._reload_connection()
 581             #delete VM ports attached to this networks before the network
 582             ports = self.neutron.list_ports(network_id=net_id)
 583             for p in ports['ports']:
 584                 try:
 585                     self.neutron.delete_port(p["id"])
 586                 except Exception as e:
 587                     self.logger.error("Error deleting port %s: %s", p["id"], str(e))
 588             self.neutron.delete_network(net_id)
 589             return net_id
 590         except (neExceptions.ConnectionFailed, neExceptions.NetworkNotFoundClient, neExceptions.NeutronException,
 591                 ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 592             self._format_exception(e)
 593
 594     def refresh_nets_status(self, net_list):
 595         '''Get the status of the networks
 596            Params: the list of network identifiers
 597            Returns a dictionary with:
 598                 net_id:         #VIM id of this network
 599                     status:     #Mandatory. Text with one of:
 600                                 #  DELETED (not found at vim)
 601                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 602                                 #  OTHER (Vim reported other status not understood)
 603                                 #  ERROR (VIM indicates an ERROR status)
 604                                 #  ACTIVE, INACTIVE, DOWN (admin down),
 605                                 #  BUILD (on building process)
 606                                 #
 607                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 608                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 609
 610         '''
 611         net_dict={}
 612         for net_id in net_list:
 613             net = {}
 614             try:
 615                 net_vim = self.get_network(net_id)
 616                 if net_vim['status'] in netStatus2manoFormat:
 617                     net["status"] = netStatus2manoFormat[ net_vim['status'] ]
 618                 else:
 619                     net["status"] = "OTHER"
 620                     net["error_msg"] = "VIM status reported " + net_vim['status']
 621
 622                 if net['status'] == "ACTIVE" and not net_vim['admin_state_up']:
 623                     net['status'] = 'DOWN'
 624
 625                 net['vim_info'] = self.serialize(net_vim)
 626
 627                 if net_vim.get('fault'):  #TODO
 628                     net['error_msg'] = str(net_vim['fault'])
 629             except vimconn.vimconnNotFoundException as e:
 630                 self.logger.error("Exception getting net status: %s", str(e))
 631                 net['status'] = "DELETED"
 632                 net['error_msg'] = str(e)
 633             except vimconn.vimconnException as e:
 634                 self.logger.error("Exception getting net status: %s", str(e))
 635                 net['status'] = "VIM_ERROR"
 636                 net['error_msg'] = str(e)
 637             net_dict[net_id] = net
 638         return net_dict
 639
 640     def get_flavor(self, flavor_id):
 641         '''Obtain flavor details from the  VIM. Returns the flavor dict details'''
 642         self.logger.debug("Getting flavor '%s'", flavor_id)
 643         try:
 644             self._reload_connection()
 645             flavor = self.nova.flavors.find(id=flavor_id)
 646             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 647             return flavor.to_dict()
 648         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 649             self._format_exception(e)
 650
 651     def get_flavor_id_from_data(self, flavor_dict):
 652         """Obtain flavor id that match the flavor description
 653            Returns the flavor_id or raises a vimconnNotFoundException
 654            flavor_dict: contains the required ram, vcpus, disk
 655            If 'use_existing_flavors' is set to True at config, the closer flavor that provides same or more ram, vcpus
 656                 and disk is returned. Otherwise a flavor with exactly same ram, vcpus and disk is returned or a
 657                 vimconnNotFoundException is raised
 658         """
 659         exact_match = False if self.config.get('use_existing_flavors') else True
 660         try:
 661             self._reload_connection()
 662             flavor_candidate_id = None
 663             flavor_candidate_data = (10000, 10000, 10000)
 664             flavor_target = (flavor_dict["ram"], flavor_dict["vcpus"], flavor_dict["disk"])
 665             # numa=None
 666             numas = flavor_dict.get("extended", {}).get("numas")
 667             if numas:
 668                 #TODO
 669                 raise vimconn.vimconnNotFoundException("Flavor with EPA still not implemted")
 670                 # if len(numas) > 1:
 671                 #     raise vimconn.vimconnNotFoundException("Cannot find any flavor with more than one numa")
 672                 # numa=numas[0]
 673                 # numas = extended.get("numas")
 674             for flavor in self.nova.flavors.list():
 675                 epa = flavor.get_keys()
 676                 if epa:
 677                     continue
 678                     # TODO
 679                 flavor_data = (flavor.ram, flavor.vcpus, flavor.disk)
 680                 if flavor_data == flavor_target:
 681                     return flavor.id
 682                 elif not exact_match and flavor_target < flavor_data < flavor_candidate_data:
 683                     flavor_candidate_id = flavor.id
 684                     flavor_candidate_data = flavor_data
 685             if not exact_match and flavor_candidate_id:
 686                 return flavor_candidate_id
 687             raise vimconn.vimconnNotFoundException("Cannot find any flavor matching '{}'".format(str(flavor_dict)))
 688         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 689             self._format_exception(e)
 690
 691     def new_flavor(self, flavor_data, change_name_if_used=True):
 692         '''Adds a tenant flavor to openstack VIM
 693         if change_name_if_used is True, it will change name in case of conflict, because it is not supported name repetition
 694         Returns the flavor identifier
 695         '''
 696         self.logger.debug("Adding flavor '%s'", str(flavor_data))
 697         retry=0
 698         max_retries=3
 699         name_suffix = 0
 700         try:
 701             name=flavor_data['name']
 702             while retry<max_retries:
 703                 retry+=1
 704                 try:
 705                     self._reload_connection()
 706                     if change_name_if_used:
 707                         #get used names
 708                         fl_names=[]
 709                         fl=self.nova.flavors.list()
 710                         for f in fl:
 711                             fl_names.append(f.name)
 712                         while name in fl_names:
 713                             name_suffix += 1
 714                             name = flavor_data['name']+"-" + str(name_suffix)
 715
 716                     ram = flavor_data.get('ram',64)
 717                     vcpus = flavor_data.get('vcpus',1)
 718                     numa_properties=None
 719
 720                     extended = flavor_data.get("extended")
 721                     if extended:
 722                         numas=extended.get("numas")
 723                         if numas:
 724                             numa_nodes = len(numas)
 725                             if numa_nodes > 1:
 726                                 return -1, "Can not add flavor with more than one numa"
 727                             numa_properties = {"hw:numa_nodes":str(numa_nodes)}
 728                             numa_properties["hw:mem_page_size"] = "large"
 729                             numa_properties["hw:cpu_policy"] = "dedicated"
 730                             numa_properties["hw:numa_mempolicy"] = "strict"
 731                             if self.vim_type == "VIO":
 732                                 numa_properties["vmware:extra_config"] = '{"numa.nodeAffinity":"0"}'
 733                                 numa_properties["vmware:latency_sensitivity_level"] = "high"
 734                             for numa in numas:
 735                                 #overwrite ram and vcpus
 736                                 #check if key 'memory' is present in numa else use ram value at flavor
 737                                 if 'memory' in numa:
 738                                     ram = numa['memory']*1024
 739                                 #See for reference: https://specs.openstack.org/openstack/nova-specs/specs/mitaka/implemented/virt-driver-cpu-thread-pinning.html
 740                                 if 'paired-threads' in numa:
 741                                     vcpus = numa['paired-threads']*2
 742                                     #cpu_thread_policy "require" implies that the compute node must have an STM architecture
 743                                     numa_properties["hw:cpu_thread_policy"] = "require"
 744                                     numa_properties["hw:cpu_policy"] = "dedicated"
 745                                 elif 'cores' in numa:
 746                                     vcpus = numa['cores']
 747                                     # cpu_thread_policy "prefer" implies that the host must not have an SMT architecture, or a non-SMT architecture will be emulated
 748                                     numa_properties["hw:cpu_thread_policy"] = "isolate"
 749                                     numa_properties["hw:cpu_policy"] = "dedicated"
 750                                 elif 'threads' in numa:
 751                                     vcpus = numa['threads']
 752                                     # cpu_thread_policy "prefer" implies that the host may or may not have an SMT architecture
 753                                     numa_properties["hw:cpu_thread_policy"] = "prefer"
 754                                     numa_properties["hw:cpu_policy"] = "dedicated"
 755                                 # for interface in numa.get("interfaces",() ):
 756                                 #     if interface["dedicated"]=="yes":
 757                                 #         raise vimconn.vimconnException("Passthrough interfaces are not supported for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable)
 758                                 #     #TODO, add the key 'pci_passthrough:alias"="<label at config>:<number ifaces>"' when a way to connect it is available
 759
 760                     #create flavor
 761                     new_flavor=self.nova.flavors.create(name,
 762                                     ram,
 763                                     vcpus,
 764                                     flavor_data.get('disk',0),
 765                                     is_public=flavor_data.get('is_public', True)
 766                                 )
 767                     #add metadata
 768                     if numa_properties:
 769                         new_flavor.set_keys(numa_properties)
 770                     return new_flavor.id
 771                 except nvExceptions.Conflict as e:
 772                     if change_name_if_used and retry < max_retries:
 773                         continue
 774                     self._format_exception(e)
 775         #except nvExceptions.BadRequest as e:
 776         except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError, KeyError) as e:
 777             self._format_exception(e)
 778
 779     def delete_flavor(self,flavor_id):
 780         '''Deletes a tenant flavor from openstack VIM. Returns the old flavor_id
 781         '''
 782         try:
 783             self._reload_connection()
 784             self.nova.flavors.delete(flavor_id)
 785             return flavor_id
 786         #except nvExceptions.BadRequest as e:
 787         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 788             self._format_exception(e)
 789
 790     def new_image(self,image_dict):
 791         '''
 792         Adds a tenant image to VIM. imge_dict is a dictionary with:
 793             name: name
 794             disk_format: qcow2, vhd, vmdk, raw (by default), ...
 795             location: path or URI
 796             public: "yes" or "no"
 797             metadata: metadata of the image
 798         Returns the image_id
 799         '''
 800         retry=0
 801         max_retries=3
 802         while retry<max_retries:
 803             retry+=1
 804             try:
 805                 self._reload_connection()
 806                 #determine format  http://docs.openstack.org/developer/glance/formats.html
 807                 if "disk_format" in image_dict:
 808                     disk_format=image_dict["disk_format"]
 809                 else: #autodiscover based on extension
 810                     if image_dict['location'].endswith(".qcow2"):
 811                         disk_format="qcow2"
 812                     elif image_dict['location'].endswith(".vhd"):
 813                         disk_format="vhd"
 814                     elif image_dict['location'].endswith(".vmdk"):
 815                         disk_format="vmdk"
 816                     elif image_dict['location'].endswith(".vdi"):
 817                         disk_format="vdi"
 818                     elif image_dict['location'].endswith(".iso"):
 819                         disk_format="iso"
 820                     elif image_dict['location'].endswith(".aki"):
 821                         disk_format="aki"
 822                     elif image_dict['location'].endswith(".ari"):
 823                         disk_format="ari"
 824                     elif image_dict['location'].endswith(".ami"):
 825                         disk_format="ami"
 826                     else:
 827                         disk_format="raw"
 828                 self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location'])
 829                 if self.vim_type == "VIO":
 830                     container_format = "bare"
 831                     if 'container_format' in image_dict:
 832                         container_format = image_dict['container_format']
 833                     new_image = self.glance.images.create(name=image_dict['name'], container_format=container_format,
 834                                                           disk_format=disk_format)
 835                 else:
 836                     new_image = self.glance.images.create(name=image_dict['name'])
 837                 if image_dict['location'].startswith("http"):
 838                     # TODO there is not a method to direct download. It must be downloaded locally with requests
 839                     raise vimconn.vimconnNotImplemented("Cannot create image from URL")
 840                 else: #local path
 841                     with open(image_dict['location']) as fimage:
 842                         self.glance.images.upload(new_image.id, fimage)
 843                         #new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 844                         #    container_format="bare", data=fimage, disk_format=disk_format)
 845                 metadata_to_load = image_dict.get('metadata')
 846                 # TODO location is a reserved word for current openstack versions. fixed for VIO please check for openstack
 847                 if self.vim_type == "VIO":
 848                     metadata_to_load['upload_location'] = image_dict['location']
 849                 else:
 850                     metadata_to_load['location'] = image_dict['location']
 851                 self.glance.images.update(new_image.id, **metadata_to_load)
 852                 return new_image.id
 853             except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e:
 854                 self._format_exception(e)
 855             except (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 856                 if retry==max_retries:
 857                     continue
 858                 self._format_exception(e)
 859             except IOError as e:  #can not open the file
 860                 raise vimconn.vimconnConnectionException(type(e).__name__ + ": " + str(e)+ " for " + image_dict['location'],
 861                                                          http_code=vimconn.HTTP_Bad_Request)
 862
 863     def delete_image(self, image_id):
 864         '''Deletes a tenant image from openstack VIM. Returns the old id
 865         '''
 866         try:
 867             self._reload_connection()
 868             self.glance.images.delete(image_id)
 869             return image_id
 870         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, gl1Exceptions.HTTPNotFound, ConnectionError) as e: #TODO remove
 871             self._format_exception(e)
 872
 873     def get_image_id_from_path(self, path):
 874         '''Get the image id from image path in the VIM database. Returns the image_id'''
 875         try:
 876             self._reload_connection()
 877             images = self.glance.images.list()
 878             for image in images:
 879                 if image.metadata.get("location")==path:
 880                     return image.id
 881             raise vimconn.vimconnNotFoundException("image with location '{}' not found".format( path))
 882         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 883             self._format_exception(e)
 884
 885     def get_image_list(self, filter_dict={}):
 886         '''Obtain tenant images from VIM
 887         Filter_dict can be:
 888             id: image id
 889             name: image name
 890             checksum: image checksum
 891         Returns the image list of dictionaries:
 892             [{<the fields at Filter_dict plus some VIM specific>}, ...]
 893             List can be empty
 894         '''
 895         self.logger.debug("Getting image list from VIM filter: '%s'", str(filter_dict))
 896         try:
 897             self._reload_connection()
 898             filter_dict_os = filter_dict.copy()
 899             #First we filter by the available filter fields: name, id. The others are removed.
 900             image_list = self.glance.images.list()
 901             filtered_list = []
 902             for image in image_list:
 903                 try:
 904                     if filter_dict.get("name") and image["name"] != filter_dict["name"]:
 905                         continue
 906                     if filter_dict.get("id") and image["id"] != filter_dict["id"]:
 907                         continue
 908                     if filter_dict.get("checksum") and image["checksum"] != filter_dict["checksum"]:
 909                         continue
 910
 911                     filtered_list.append(image.copy())
 912                 except gl1Exceptions.HTTPNotFound:
 913                     pass
 914             return filtered_list
 915         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 916             self._format_exception(e)
 917
 918     def __wait_for_vm(self, vm_id, status):
 919         """wait until vm is in the desired status and return True.
 920         If the VM gets in ERROR status, return false.
 921         If the timeout is reached generate an exception"""
 922         elapsed_time = 0
 923         while elapsed_time < server_timeout:
 924             vm_status = self.nova.servers.get(vm_id).status
 925             if vm_status == status:
 926                 return True
 927             if vm_status == 'ERROR':
 928                 return False
 929             time.sleep(5)
 930             elapsed_time += 5
 931
 932         # if we exceeded the timeout rollback
 933         if elapsed_time >= server_timeout:
 934             raise vimconn.vimconnException('Timeout waiting for instance ' + vm_id + ' to get ' + status,
 935                                            http_code=vimconn.HTTP_Request_Timeout)
 936
 937     def _get_openstack_availablity_zones(self):
 938         """
 939         Get from openstack availability zones available
 940         :return:
 941         """
 942         try:
 943             openstack_availability_zone = self.nova.availability_zones.list()
 944             openstack_availability_zone = [str(zone.zoneName) for zone in openstack_availability_zone
 945                                            if zone.zoneName != 'internal']
 946             return openstack_availability_zone
 947         except Exception as e:
 948             return None
 949
 950     def _set_availablity_zones(self):
 951         """
 952         Set vim availablity zone
 953         :return:
 954         """
 955
 956         if 'availability_zone' in self.config:
 957             vim_availability_zones = self.config.get('availability_zone')
 958             if isinstance(vim_availability_zones, str):
 959                 self.availability_zone = [vim_availability_zones]
 960             elif isinstance(vim_availability_zones, list):
 961                 self.availability_zone = vim_availability_zones
 962         else:
 963             self.availability_zone = self._get_openstack_availablity_zones()
 964
 965     def _get_vm_availability_zone(self, availability_zone_index, availability_zone_list):
 966         """
 967         Return thge availability zone to be used by the created VM.
 968         :return: The VIM availability zone to be used or None
 969         """
 970         if availability_zone_index is None:
 971             if not self.config.get('availability_zone'):
 972                 return None
 973             elif isinstance(self.config.get('availability_zone'), str):
 974                 return self.config['availability_zone']
 975             else:
 976                 # TODO consider using a different parameter at config for default AV and AV list match
 977                 return self.config['availability_zone'][0]
 978
 979         vim_availability_zones = self.availability_zone
 980         # check if VIM offer enough availability zones describe in the VNFD
 981         if vim_availability_zones and len(availability_zone_list) <= len(vim_availability_zones):
 982             # check if all the names of NFV AV match VIM AV names
 983             match_by_index = False
 984             for av in availability_zone_list:
 985                 if av not in vim_availability_zones:
 986                     match_by_index = True
 987                     break
 988             if match_by_index:
 989                 return vim_availability_zones[availability_zone_index]
 990             else:
 991                 return availability_zone_list[availability_zone_index]
 992         else:
 993             raise vimconn.vimconnConflictException("No enough availability zones at VIM for this deployment")
 994
 995     def new_vminstance(self, name, description, start, image_id, flavor_id, net_list, cloud_config=None, disk_list=None,
 996                        availability_zone_index=None, availability_zone_list=None):
 997         """Adds a VM instance to VIM
 998         Params:
 999             start: indicates if VM must start or boot in pause mode. Ignored
1000             image_id,flavor_id: iamge and flavor uuid
1001             net_list: list of interfaces, each one is a dictionary with:
1002                 name:
1003                 net_id: network uuid to connect
1004                 vpci: virtual vcpi to assign, ignored because openstack lack #TODO
1005                 model: interface model, ignored #TODO
1006                 mac_address: used for  SR-IOV ifaces #TODO for other types
1007                 use: 'data', 'bridge',  'mgmt'
1008                 type: 'virtual', 'PCI-PASSTHROUGH'('PF'), 'SR-IOV'('VF'), 'VFnotShared'
1009                 vim_id: filled/added by this function
1010                 floating_ip: True/False (or it can be None)
1011             'cloud_config': (optional) dictionary with:
1012             'key-pairs': (optional) list of strings with the public key to be inserted to the default user
1013             'users': (optional) list of users to be inserted, each item is a dict with:
1014                 'name': (mandatory) user name,
1015                 'key-pairs': (optional) list of strings with the public key to be inserted to the user
1016             'user-data': (optional) string is a text script to be passed directly to cloud-init
1017             'config-files': (optional). List of files to be transferred. Each item is a dict with:
1018                 'dest': (mandatory) string with the destination absolute path
1019                 'encoding': (optional, by default text). Can be one of:
1020                     'b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64'
1021                 'content' (mandatory): string with the content of the file
1022                 'permissions': (optional) string with file permissions, typically octal notation '0644'
1023                 'owner': (optional) file owner, string with the format 'owner:group'
1024             'boot-data-drive': boolean to indicate if user-data must be passed using a boot drive (hard disk)
1025             'disk_list': (optional) list with additional disks to the VM. Each item is a dict with:
1026                 'image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted
1027                 'size': (mandatory) string with the size of the disk in GB
1028                 'vim_id' (optional) should use this existing volume id
1029             availability_zone_index: Index of availability_zone_list to use for this this VM. None if not AV required
1030             availability_zone_list: list of availability zones given by user in the VNFD descriptor.  Ignore if
1031                 availability_zone_index is None
1032                 #TODO ip, security groups
1033         Returns a tuple with the instance identifier and created_items or raises an exception on error
1034             created_items can be None or a dictionary where this method can include key-values that will be passed to
1035             the method delete_vminstance and action_vminstance. Can be used to store created ports, volumes, etc.
1036             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
1037             as not present.
1038         """
1039         self.logger.debug("new_vminstance input: image='%s' flavor='%s' nics='%s'",image_id, flavor_id,str(net_list))
1040         try:
1041             server = None
1042             created_items = {}
1043             # metadata = {}
1044             net_list_vim = []
1045             external_network = []   # list of external networks to be connected to instance, later on used to create floating_ip
1046             no_secured_ports = []   # List of port-is with port-security disabled
1047             self._reload_connection()
1048             # metadata_vpci = {}   # For a specific neutron plugin
1049             block_device_mapping = None
1050             for net in net_list:
1051                 if not net.get("net_id"):   # skip non connected iface
1052                     continue
1053
1054                 port_dict={
1055                     "network_id": net["net_id"],
1056                     "name": net.get("name"),
1057                     "admin_state_up": True
1058                 }
1059                 if net["type"]=="virtual":
1060                     pass
1061                     # if "vpci" in net:
1062                     #     metadata_vpci[ net["net_id"] ] = [[ net["vpci"], "" ]]
1063                 elif net["type"] == "VF" or net["type"] == "SR-IOV":  # for VF
1064                     # if "vpci" in net:
1065                     #     if "VF" not in metadata_vpci:
1066                     #         metadata_vpci["VF"]=[]
1067                     #     metadata_vpci["VF"].append([ net["vpci"], "" ])
1068                     port_dict["binding:vnic_type"]="direct"
1069                     # VIO specific Changes
1070                     if self.vim_type == "VIO":
1071                         # Need to create port with port_security_enabled = False and no-security-groups
1072                         port_dict["port_security_enabled"]=False
1073                         port_dict["provider_security_groups"]=[]
1074                         port_dict["security_groups"]=[]
1075                 else:   # For PT PCI-PASSTHROUGH
1076                     # VIO specific Changes
1077                     # Current VIO release does not support port with type 'direct-physical'
1078                     # So no need to create virtual port in case of PCI-device.
1079                     # Will update port_dict code when support gets added in next VIO release
1080                     if self.vim_type == "VIO":
1081                         raise vimconn.vimconnNotSupportedException(
1082                             "Current VIO release does not support full passthrough (PT)")
1083                     # if "vpci" in net:
1084                     #     if "PF" not in metadata_vpci:
1085                     #         metadata_vpci["PF"]=[]
1086                     #     metadata_vpci["PF"].append([ net["vpci"], "" ])
1087                     port_dict["binding:vnic_type"]="direct-physical"
1088                 if not port_dict["name"]:
1089                     port_dict["name"]=name
1090                 if net.get("mac_address"):
1091                     port_dict["mac_address"]=net["mac_address"]
1092                 if net.get("ip_address"):
1093                     port_dict["fixed_ips"] = [{'ip_address': net["ip_address"]}]
1094                     # TODO add 'subnet_id': <subnet_id>
1095                 new_port = self.neutron.create_port({"port": port_dict })
1096                 created_items["port:" + str(new_port["port"]["id"])] = True
1097                 net["mac_adress"] = new_port["port"]["mac_address"]
1098                 net["vim_id"] = new_port["port"]["id"]
1099                 # if try to use a network without subnetwork, it will return a emtpy list
1100                 fixed_ips = new_port["port"].get("fixed_ips")
1101                 if fixed_ips:
1102                     net["ip"] = fixed_ips[0].get("ip_address")
1103                 else:
1104                     net["ip"] = None
1105
1106                 port = {"port-id": new_port["port"]["id"]}
1107                 if float(self.nova.api_version.get_string()) >= 2.32:
1108                     port["tag"] = new_port["port"]["name"]
1109                 net_list_vim.append(port)
1110
1111                 if net.get('floating_ip', False):
1112                     net['exit_on_floating_ip_error'] = True
1113                     external_network.append(net)
1114                 elif net['use'] == 'mgmt' and self.config.get('use_floating_ip'):
1115                     net['exit_on_floating_ip_error'] = False
1116                     external_network.append(net)
1117                     net['floating_ip'] = self.config.get('use_floating_ip')
1118
1119                 # If port security is disabled when the port has not yet been attached to the VM, then all vm traffic is dropped.
1120                 # As a workaround we wait until the VM is active and then disable the port-security
1121                 if net.get("port_security") == False and not self.config.get("no_port_security_extension"):
1122                     no_secured_ports.append(new_port["port"]["id"])
1123
1124             # if metadata_vpci:
1125             #     metadata = {"pci_assignement": json.dumps(metadata_vpci)}
1126             #     if len(metadata["pci_assignement"]) >255:
1127             #         #limit the metadata size
1128             #         #metadata["pci_assignement"] = metadata["pci_assignement"][0:255]
1129             #         self.logger.warn("Metadata deleted since it exceeds the expected length (255) ")
1130             #         metadata = {}
1131
1132             self.logger.debug("name '%s' image_id '%s'flavor_id '%s' net_list_vim '%s' description '%s'",
1133                               name, image_id, flavor_id, str(net_list_vim), description)
1134
1135             security_groups = self.config.get('security_groups')
1136             if type(security_groups) is str:
1137                 security_groups = ( security_groups, )
1138             # cloud config
1139             config_drive, userdata = self._create_user_data(cloud_config)
1140
1141             # Create additional volumes in case these are present in disk_list
1142             base_disk_index = ord('b')
1143             if disk_list:
1144                 block_device_mapping = {}
1145                 for disk in disk_list:
1146                     if disk.get('vim_id'):
1147                         block_device_mapping['_vd' + chr(base_disk_index)] = disk['vim_id']
1148                     else:
1149                         if 'image_id' in disk:
1150                             volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1151                                                                 chr(base_disk_index), imageRef=disk['image_id'])
1152                         else:
1153                             volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1154                                                                 chr(base_disk_index))
1155                         created_items["volume:" + str(volume.id)] = True
1156                         block_device_mapping['_vd' + chr(base_disk_index)] = volume.id
1157                     base_disk_index += 1
1158
1159                 # Wait until created volumes are with status available
1160                 elapsed_time = 0
1161                 while elapsed_time < volume_timeout:
1162                     for created_item in created_items:
1163                         v, _, volume_id = created_item.partition(":")
1164                         if v == 'volume':
1165                             if self.cinder.volumes.get(volume_id).status != 'available':
1166                                 break
1167                     else:  # all ready: break from while
1168                         break
1169                     time.sleep(5)
1170                     elapsed_time += 5
1171                 # If we exceeded the timeout rollback
1172                 if elapsed_time >= volume_timeout:
1173                     raise vimconn.vimconnException('Timeout creating volumes for instance ' + name,
1174                                                    http_code=vimconn.HTTP_Request_Timeout)
1175             # get availability Zone
1176             vm_av_zone = self._get_vm_availability_zone(availability_zone_index, availability_zone_list)
1177
1178             self.logger.debug("nova.servers.create({}, {}, {}, nics={}, security_groups={}, "
1179                               "availability_zone={}, key_name={}, userdata={}, config_drive={}, "
1180                               "block_device_mapping={})".format(name, image_id, flavor_id, net_list_vim,
1181                                                                 security_groups, vm_av_zone, self.config.get('keypair'),
1182                                                                 userdata, config_drive, block_device_mapping))
1183             server = self.nova.servers.create(name, image_id, flavor_id, nics=net_list_vim,
1184                                               security_groups=security_groups,
1185                                               availability_zone=vm_av_zone,
1186                                               key_name=self.config.get('keypair'),
1187                                               userdata=userdata,
1188                                               config_drive=config_drive,
1189                                               block_device_mapping=block_device_mapping
1190                                               )  # , description=description)
1191
1192             vm_start_time = time.time()
1193             # Previously mentioned workaround to wait until the VM is active and then disable the port-security
1194             if no_secured_ports:
1195                 self.__wait_for_vm(server.id, 'ACTIVE')
1196
1197             for port_id in no_secured_ports:
1198                 try:
1199                     self.neutron.update_port(port_id,
1200                                              {"port": {"port_security_enabled": False, "security_groups": None}})
1201                 except Exception as e:
1202                     raise vimconn.vimconnException("It was not possible to disable port security for port {}".format(
1203                         port_id))
1204             # print "DONE :-)", server
1205
1206             # pool_id = None
1207             if external_network:
1208                 floating_ips = self.neutron.list_floatingips().get("floatingips", ())
1209             for floating_network in external_network:
1210                 try:
1211                     assigned = False
1212                     while not assigned:
1213                         if floating_ips:
1214                             ip = floating_ips.pop(0)
1215                             if ip.get("port_id", False) or ip.get('tenant_id') != server.tenant_id:
1216                                 continue
1217                             if isinstance(floating_network['floating_ip'], str):
1218                                 if ip.get("floating_network_id") != floating_network['floating_ip']:
1219                                     continue
1220                             free_floating_ip = ip.get("floating_ip_address")
1221                         else:
1222                             if isinstance(floating_network['floating_ip'], str) and \
1223                                 floating_network['floating_ip'].lower() != "true":
1224                                 pool_id = floating_network['floating_ip']
1225                             else:
1226                                 # Find the external network
1227                                 external_nets = list()
1228                                 for net in self.neutron.list_networks()['networks']:
1229                                     if net['router:external']:
1230                                             external_nets.append(net)
1231
1232                                 if len(external_nets) == 0:
1233                                     raise vimconn.vimconnException("Cannot create floating_ip automatically since no external "
1234                                                                    "network is present",
1235                                                                     http_code=vimconn.HTTP_Conflict)
1236                                 if len(external_nets) > 1:
1237                                     raise vimconn.vimconnException("Cannot create floating_ip automatically since multiple "
1238                                                                    "external networks are present",
1239                                                                    http_code=vimconn.HTTP_Conflict)
1240
1241                                 pool_id = external_nets[0].get('id')
1242                             param = {'floatingip': {'floating_network_id': pool_id, 'tenant_id': server.tenant_id}}
1243                             try:
1244                                 # self.logger.debug("Creating floating IP")
1245                                 new_floating_ip = self.neutron.create_floatingip(param)
1246                                 free_floating_ip = new_floating_ip['floatingip']['floating_ip_address']
1247                             except Exception as e:
1248                                 raise vimconn.vimconnException(type(e).__name__ + ": Cannot create new floating_ip " +
1249                                                                str(e), http_code=vimconn.HTTP_Conflict)
1250
1251                         fix_ip = floating_network.get('ip')
1252                         while not assigned:
1253                             try:
1254                                 server.add_floating_ip(free_floating_ip, fix_ip)
1255                                 assigned = True
1256                             except Exception as e:
1257                                 # openstack need some time after VM creation to asign an IP. So retry if fails
1258                                 vm_status = self.nova.servers.get(server.id).status
1259                                 if vm_status != 'ACTIVE' and vm_status != 'ERROR':
1260                                     if time.time() - vm_start_time < server_timeout:
1261                                         time.sleep(5)
1262                                         continue
1263                                 raise vimconn.vimconnException(
1264                                     "Cannot create floating_ip: {} {}".format(type(e).__name__, e),
1265                                     http_code=vimconn.HTTP_Conflict)
1266
1267                 except Exception as e:
1268                     if not floating_network['exit_on_floating_ip_error']:
1269                         self.logger.warn("Cannot create floating_ip. %s", str(e))
1270                         continue
1271                     raise
1272
1273             return server.id, created_items
1274 #        except nvExceptions.NotFound as e:
1275 #            error_value=-vimconn.HTTP_Not_Found
1276 #            error_text= "vm instance %s not found" % vm_id
1277 #        except TypeError as e:
1278 #            raise vimconn.vimconnException(type(e).__name__ + ": "+  str(e), http_code=vimconn.HTTP_Bad_Request)
1279
1280         except Exception as e:
1281             server_id = None
1282             if server:
1283                 server_id = server.id
1284             try:
1285                 self.delete_vminstance(server_id, created_items)
1286             except Exception as e2:
1287                 self.logger.error("new_vminstance rollback fail {}".format(e2))
1288
1289             self._format_exception(e)
1290
1291     def get_vminstance(self,vm_id):
1292         '''Returns the VM instance information from VIM'''
1293         #self.logger.debug("Getting VM from VIM")
1294         try:
1295             self._reload_connection()
1296             server = self.nova.servers.find(id=vm_id)
1297             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
1298             return server.to_dict()
1299         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1300             self._format_exception(e)
1301
1302     def get_vminstance_console(self,vm_id, console_type="vnc"):
1303         '''
1304         Get a console for the virtual machine
1305         Params:
1306             vm_id: uuid of the VM
1307             console_type, can be:
1308                 "novnc" (by default), "xvpvnc" for VNC types,
1309                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
1310         Returns dict with the console parameters:
1311                 protocol: ssh, ftp, http, https, ...
1312                 server:   usually ip address
1313                 port:     the http, ssh, ... port
1314                 suffix:   extra text, e.g. the http path and query string
1315         '''
1316         self.logger.debug("Getting VM CONSOLE from VIM")
1317         try:
1318             self._reload_connection()
1319             server = self.nova.servers.find(id=vm_id)
1320             if console_type == None or console_type == "novnc":
1321                 console_dict = server.get_vnc_console("novnc")
1322             elif console_type == "xvpvnc":
1323                 console_dict = server.get_vnc_console(console_type)
1324             elif console_type == "rdp-html5":
1325                 console_dict = server.get_rdp_console(console_type)
1326             elif console_type == "spice-html5":
1327                 console_dict = server.get_spice_console(console_type)
1328             else:
1329                 raise vimconn.vimconnException("console type '{}' not allowed".format(console_type), http_code=vimconn.HTTP_Bad_Request)
1330
1331             console_dict1 = console_dict.get("console")
1332             if console_dict1:
1333                 console_url = console_dict1.get("url")
1334                 if console_url:
1335                     #parse console_url
1336                     protocol_index = console_url.find("//")
1337                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1338                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1339                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1340                         return -vimconn.HTTP_Internal_Server_Error, "Unexpected response from VIM"
1341                     console_dict={"protocol": console_url[0:protocol_index],
1342                                   "server":   console_url[protocol_index+2:port_index],
1343                                   "port":     console_url[port_index:suffix_index],
1344                                   "suffix":   console_url[suffix_index+1:]
1345                                   }
1346                     protocol_index += 2
1347                     return console_dict
1348             raise vimconn.vimconnUnexpectedResponse("Unexpected response from VIM")
1349
1350         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.BadRequest, ConnectionError) as e:
1351             self._format_exception(e)
1352
1353     def delete_vminstance(self, vm_id, created_items=None):
1354         '''Removes a VM instance from VIM. Returns the old identifier
1355         '''
1356         #print "osconnector: Getting VM from VIM"
1357         if created_items == None:
1358             created_items = {}
1359         try:
1360             self._reload_connection()
1361             # delete VM ports attached to this networks before the virtual machine
1362             for k, v in created_items.items():
1363                 if not v:  # skip already deleted
1364                     continue
1365                 try:
1366                     k_item, _, k_id = k.partition(":")
1367                     if k_item == "port":
1368                         self.neutron.delete_port(k_id)
1369                 except Exception as e:
1370                     self.logger.error("Error deleting port: {}: {}".format(type(e).__name__, e))
1371
1372             # #commented because detaching the volumes makes the servers.delete not work properly ?!?
1373             # #dettach volumes attached
1374             # server = self.nova.servers.get(vm_id)
1375             # volumes_attached_dict = server._info['os-extended-volumes:volumes_attached']   #volume['id']
1376             # #for volume in volumes_attached_dict:
1377             # #    self.cinder.volumes.detach(volume['id'])
1378
1379             if vm_id:
1380                 self.nova.servers.delete(vm_id)
1381
1382             # delete volumes. Although having detached, they should have in active status before deleting
1383             # we ensure in this loop
1384             keep_waiting = True
1385             elapsed_time = 0
1386             while keep_waiting and elapsed_time < volume_timeout:
1387                 keep_waiting = False
1388                 for k, v in created_items.items():
1389                     if not v:  # skip already deleted
1390                         continue
1391                     try:
1392                         k_item, _, k_id = k.partition(":")
1393                         if k_item == "volume":
1394                             if self.cinder.volumes.get(k_id).status != 'available':
1395                                 keep_waiting = True
1396                             else:
1397                                 self.cinder.volumes.delete(k_id)
1398                     except Exception as e:
1399                         self.logger.error("Error deleting volume: {}: {}".format(type(e).__name__, e))
1400                 if keep_waiting:
1401                     time.sleep(1)
1402                     elapsed_time += 1
1403             return None
1404         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
1405             self._format_exception(e)
1406
1407     def refresh_vms_status(self, vm_list):
1408         '''Get the status of the virtual machines and their interfaces/ports
1409            Params: the list of VM identifiers
1410            Returns a dictionary with:
1411                 vm_id:          #VIM id of this Virtual Machine
1412                     status:     #Mandatory. Text with one of:
1413                                 #  DELETED (not found at vim)
1414                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1415                                 #  OTHER (Vim reported other status not understood)
1416                                 #  ERROR (VIM indicates an ERROR status)
1417                                 #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
1418                                 #  CREATING (on building process), ERROR
1419                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
1420                                 #
1421                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
1422                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
1423                     interfaces:
1424                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
1425                         mac_address:      #Text format XX:XX:XX:XX:XX:XX
1426                         vim_net_id:       #network id where this interface is connected
1427                         vim_interface_id: #interface/port VIM id
1428                         ip_address:       #null, or text with IPv4, IPv6 address
1429                         compute_node:     #identification of compute node where PF,VF interface is allocated
1430                         pci:              #PCI address of the NIC that hosts the PF,VF
1431                         vlan:             #physical VLAN used for VF
1432         '''
1433         vm_dict={}
1434         self.logger.debug("refresh_vms status: Getting tenant VM instance information from VIM")
1435         for vm_id in vm_list:
1436             vm={}
1437             try:
1438                 vm_vim = self.get_vminstance(vm_id)
1439                 if vm_vim['status'] in vmStatus2manoFormat:
1440                     vm['status']    =  vmStatus2manoFormat[ vm_vim['status'] ]
1441                 else:
1442                     vm['status']    = "OTHER"
1443                     vm['error_msg'] = "VIM status reported " + vm_vim['status']
1444
1445                 vm['vim_info'] = self.serialize(vm_vim)
1446
1447                 vm["interfaces"] = []
1448                 if vm_vim.get('fault'):
1449                     vm['error_msg'] = str(vm_vim['fault'])
1450                 #get interfaces
1451                 try:
1452                     self._reload_connection()
1453                     port_dict = self.neutron.list_ports(device_id=vm_id)
1454                     for port in port_dict["ports"]:
1455                         interface={}
1456                         interface['vim_info'] = self.serialize(port)
1457                         interface["mac_address"] = port.get("mac_address")
1458                         interface["vim_net_id"] = port["network_id"]
1459                         interface["vim_interface_id"] = port["id"]
1460                         # check if OS-EXT-SRV-ATTR:host is there,
1461                         # in case of non-admin credentials, it will be missing
1462                         if vm_vim.get('OS-EXT-SRV-ATTR:host'):
1463                             interface["compute_node"] = vm_vim['OS-EXT-SRV-ATTR:host']
1464                         interface["pci"] = None
1465
1466                         # check if binding:profile is there,
1467                         # in case of non-admin credentials, it will be missing
1468                         if port.get('binding:profile'):
1469                             if port['binding:profile'].get('pci_slot'):
1470                                 # TODO: At the moment sr-iov pci addresses are converted to PF pci addresses by setting the slot to 0x00
1471                                 # TODO: This is just a workaround valid for niantinc. Find a better way to do so
1472                                 #   CHANGE DDDD:BB:SS.F to DDDD:BB:00.(F%2)   assuming there are 2 ports per nic
1473                                 pci = port['binding:profile']['pci_slot']
1474                                 # interface["pci"] = pci[:-4] + "00." + str(int(pci[-1]) % 2)
1475                                 interface["pci"] = pci
1476                         interface["vlan"] = None
1477                         #if network is of type vlan and port is of type direct (sr-iov) then set vlan id
1478                         network = self.neutron.show_network(port["network_id"])
1479                         if network['network'].get('provider:network_type') == 'vlan' and \
1480                             port.get("binding:vnic_type") == "direct":
1481                             interface["vlan"] = network['network'].get('provider:segmentation_id')
1482                         ips=[]
1483                         #look for floating ip address
1484                         try:
1485                             floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
1486                             if floating_ip_dict.get("floatingips"):
1487                                 ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address") )
1488                         except Exception:
1489                             pass
1490
1491                         for subnet in port["fixed_ips"]:
1492                             ips.append(subnet["ip_address"])
1493                         interface["ip_address"] = ";".join(ips)
1494                         vm["interfaces"].append(interface)
1495                 except Exception as e:
1496                     self.logger.error("Error getting vm interface information {}: {}".format(type(e).__name__, e),
1497                                       exc_info=True)
1498             except vimconn.vimconnNotFoundException as e:
1499                 self.logger.error("Exception getting vm status: %s", str(e))
1500                 vm['status'] = "DELETED"
1501                 vm['error_msg'] = str(e)
1502             except vimconn.vimconnException as e:
1503                 self.logger.error("Exception getting vm status: %s", str(e))
1504                 vm['status'] = "VIM_ERROR"
1505                 vm['error_msg'] = str(e)
1506             vm_dict[vm_id] = vm
1507         return vm_dict
1508
1509     def action_vminstance(self, vm_id, action_dict, created_items={}):
1510         '''Send and action over a VM instance from VIM
1511         Returns None or the console dict if the action was successfully sent to the VIM'''
1512         self.logger.debug("Action over VM '%s': %s", vm_id, str(action_dict))
1513         try:
1514             self._reload_connection()
1515             server = self.nova.servers.find(id=vm_id)
1516             if "start" in action_dict:
1517                 if action_dict["start"]=="rebuild":
1518                     server.rebuild()
1519                 else:
1520                     if server.status=="PAUSED":
1521                         server.unpause()
1522                     elif server.status=="SUSPENDED":
1523                         server.resume()
1524                     elif server.status=="SHUTOFF":
1525                         server.start()
1526             elif "pause" in action_dict:
1527                 server.pause()
1528             elif "resume" in action_dict:
1529                 server.resume()
1530             elif "shutoff" in action_dict or "shutdown" in action_dict:
1531                 server.stop()
1532             elif "forceOff" in action_dict:
1533                 server.stop() #TODO
1534             elif "terminate" in action_dict:
1535                 server.delete()
1536             elif "createImage" in action_dict:
1537                 server.create_image()
1538                 #"path":path_schema,
1539                 #"description":description_schema,
1540                 #"name":name_schema,
1541                 #"metadata":metadata_schema,
1542                 #"imageRef": id_schema,
1543                 #"disk": {"oneOf":[{"type": "null"}, {"type":"string"}] },
1544             elif "rebuild" in action_dict:
1545                 server.rebuild(server.image['id'])
1546             elif "reboot" in action_dict:
1547                 server.reboot() #reboot_type='SOFT'
1548             elif "console" in action_dict:
1549                 console_type = action_dict["console"]
1550                 if console_type == None or console_type == "novnc":
1551                     console_dict = server.get_vnc_console("novnc")
1552                 elif console_type == "xvpvnc":
1553                     console_dict = server.get_vnc_console(console_type)
1554                 elif console_type == "rdp-html5":
1555                     console_dict = server.get_rdp_console(console_type)
1556                 elif console_type == "spice-html5":
1557                     console_dict = server.get_spice_console(console_type)
1558                 else:
1559                     raise vimconn.vimconnException("console type '{}' not allowed".format(console_type),
1560                                                    http_code=vimconn.HTTP_Bad_Request)
1561                 try:
1562                     console_url = console_dict["console"]["url"]
1563                     #parse console_url
1564                     protocol_index = console_url.find("//")
1565                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1566                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1567                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1568                         raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1569                     console_dict2={"protocol": console_url[0:protocol_index],
1570                                   "server":   console_url[protocol_index+2 : port_index],
1571                                   "port":     int(console_url[port_index+1 : suffix_index]),
1572                                   "suffix":   console_url[suffix_index+1:]
1573                                   }
1574                     return console_dict2
1575                 except Exception as e:
1576                     raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1577
1578             return None
1579         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1580             self._format_exception(e)
1581         #TODO insert exception vimconn.HTTP_Unauthorized
1582
1583     ####### VIO Specific Changes #########
1584     def _genrate_vlanID(self):
1585         """
1586          Method to get unused vlanID
1587             Args:
1588                 None
1589             Returns:
1590                 vlanID
1591         """
1592         #Get used VLAN IDs
1593         usedVlanIDs = []
1594         networks = self.get_network_list()
1595         for net in networks:
1596             if net.get('provider:segmentation_id'):
1597                 usedVlanIDs.append(net.get('provider:segmentation_id'))
1598         used_vlanIDs = set(usedVlanIDs)
1599
1600         #find unused VLAN ID
1601         for vlanID_range in self.config.get('dataplane_net_vlan_range'):
1602             try:
1603                 start_vlanid , end_vlanid = map(int, vlanID_range.replace(" ", "").split("-"))
1604                 for vlanID in xrange(start_vlanid, end_vlanid + 1):
1605                     if vlanID not in used_vlanIDs:
1606                         return vlanID
1607             except Exception as exp:
1608                 raise vimconn.vimconnException("Exception {} occurred while generating VLAN ID.".format(exp))
1609         else:
1610             raise vimconn.vimconnConflictException("Unable to create the SRIOV VLAN network."\
1611                 " All given Vlan IDs {} are in use.".format(self.config.get('dataplane_net_vlan_range')))
1612
1613
1614     def _validate_vlan_ranges(self, dataplane_net_vlan_range):
1615         """
1616         Method to validate user given vlanID ranges
1617             Args:  None
1618             Returns: None
1619         """
1620         for vlanID_range in dataplane_net_vlan_range:
1621             vlan_range = vlanID_range.replace(" ", "")
1622             #validate format
1623             vlanID_pattern = r'(\d)*-(\d)*$'
1624             match_obj = re.match(vlanID_pattern, vlan_range)
1625             if not match_obj:
1626                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}.You must provide "\
1627                 "'dataplane_net_vlan_range' in format [start_ID - end_ID].".format(vlanID_range))
1628
1629             start_vlanid , end_vlanid = map(int,vlan_range.split("-"))
1630             if start_vlanid <= 0 :
1631                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1632                 "Start ID can not be zero. For VLAN "\
1633                 "networks valid IDs are 1 to 4094 ".format(vlanID_range))
1634             if end_vlanid > 4094 :
1635                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1636                 "End VLAN ID can not be greater than 4094. For VLAN "\
1637                 "networks valid IDs are 1 to 4094 ".format(vlanID_range))
1638
1639             if start_vlanid > end_vlanid:
1640                 raise vimconn.vimconnConflictException("Invalid dataplane_net_vlan_range {}."\
1641                     "You must provide a 'dataplane_net_vlan_range' in format start_ID - end_ID and "\
1642                     "start_ID < end_ID ".format(vlanID_range))
1643
1644 #NOT USED FUNCTIONS
1645
1646     def new_external_port(self, port_data):
1647         #TODO openstack if needed
1648         '''Adds a external port to VIM'''
1649         '''Returns the port identifier'''
1650         return -vimconn.HTTP_Internal_Server_Error, "osconnector.new_external_port() not implemented"
1651
1652     def connect_port_network(self, port_id, network_id, admin=False):
1653         #TODO openstack if needed
1654         '''Connects a external port to a network'''
1655         '''Returns status code of the VIM response'''
1656         return -vimconn.HTTP_Internal_Server_Error, "osconnector.connect_port_network() not implemented"
1657
1658     def new_user(self, user_name, user_passwd, tenant_id=None):
1659         '''Adds a new user to openstack VIM'''
1660         '''Returns the user identifier'''
1661         self.logger.debug("osconnector: Adding a new user to VIM")
1662         try:
1663             self._reload_connection()
1664             user=self.keystone.users.create(user_name, password=user_passwd, default_project=tenant_id)
1665             #self.keystone.tenants.add_user(self.k_creds["username"], #role)
1666             return user.id
1667         except ksExceptions.ConnectionError as e:
1668             error_value=-vimconn.HTTP_Bad_Request
1669             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1670         except ksExceptions.ClientException as e: #TODO remove
1671             error_value=-vimconn.HTTP_Bad_Request
1672             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1673         #TODO insert exception vimconn.HTTP_Unauthorized
1674         #if reaching here is because an exception
1675         self.logger.debug("new_user " + error_text)
1676         return error_value, error_text
1677
1678     def delete_user(self, user_id):
1679         '''Delete a user from openstack VIM'''
1680         '''Returns the user identifier'''
1681         if self.debug:
1682             print("osconnector: Deleting  a  user from VIM")
1683         try:
1684             self._reload_connection()
1685             self.keystone.users.delete(user_id)
1686             return 1, user_id
1687         except ksExceptions.ConnectionError as e:
1688             error_value=-vimconn.HTTP_Bad_Request
1689             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1690         except ksExceptions.NotFound as e:
1691             error_value=-vimconn.HTTP_Not_Found
1692             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1693         except ksExceptions.ClientException as e: #TODO remove
1694             error_value=-vimconn.HTTP_Bad_Request
1695             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1696         #TODO insert exception vimconn.HTTP_Unauthorized
1697         #if reaching here is because an exception
1698             self.logger.debug("delete_tenant " + error_text)
1699         return error_value, error_text
1700
1701     def get_hosts_info(self):
1702         '''Get the information of deployed hosts
1703         Returns the hosts content'''
1704         if self.debug:
1705             print("osconnector: Getting Host info from VIM")
1706         try:
1707             h_list=[]
1708             self._reload_connection()
1709             hypervisors = self.nova.hypervisors.list()
1710             for hype in hypervisors:
1711                 h_list.append( hype.to_dict() )
1712             return 1, {"hosts":h_list}
1713         except nvExceptions.NotFound as e:
1714             error_value=-vimconn.HTTP_Not_Found
1715             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1716         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1717             error_value=-vimconn.HTTP_Bad_Request
1718             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1719         #TODO insert exception vimconn.HTTP_Unauthorized
1720         #if reaching here is because an exception
1721         self.logger.debug("get_hosts_info " + error_text)
1722         return error_value, error_text
1723
1724     def get_hosts(self, vim_tenant):
1725         '''Get the hosts and deployed instances
1726         Returns the hosts content'''
1727         r, hype_dict = self.get_hosts_info()
1728         if r<0:
1729             return r, hype_dict
1730         hypervisors = hype_dict["hosts"]
1731         try:
1732             servers = self.nova.servers.list()
1733             for hype in hypervisors:
1734                 for server in servers:
1735                     if server.to_dict()['OS-EXT-SRV-ATTR:hypervisor_hostname']==hype['hypervisor_hostname']:
1736                         if 'vm' in hype:
1737                             hype['vm'].append(server.id)
1738                         else:
1739                             hype['vm'] = [server.id]
1740             return 1, hype_dict
1741         except nvExceptions.NotFound as e:
1742             error_value=-vimconn.HTTP_Not_Found
1743             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1744         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1745             error_value=-vimconn.HTTP_Bad_Request
1746             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1747         #TODO insert exception vimconn.HTTP_Unauthorized
1748         #if reaching here is because an exception
1749         self.logger.debug("get_hosts " + error_text)
1750         return error_value, error_text
1751
1752     def new_classification(self, name, ctype, definition):
1753         self.logger.debug(
1754             'Adding a new (Traffic) Classification to VIM, named %s', name)
1755         try:
1756             new_class = None
1757             self._reload_connection()
1758             if ctype not in supportedClassificationTypes:
1759                 raise vimconn.vimconnNotSupportedException(
1760                         'OpenStack VIM connector doesn\'t support provided '
1761                         'Classification Type {}, supported ones are: '
1762                         '{}'.format(ctype, supportedClassificationTypes))
1763             if not self._validate_classification(ctype, definition):
1764                 raise vimconn.vimconnException(
1765                     'Incorrect Classification definition '
1766                     'for the type specified.')
1767             classification_dict = definition
1768             classification_dict['name'] = name
1769
1770             new_class = self.neutron.create_sfc_flow_classifier(
1771                 {'flow_classifier': classification_dict})
1772             return new_class['flow_classifier']['id']
1773         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1774                 neExceptions.NeutronException, ConnectionError) as e:
1775             self.logger.error(
1776                 'Creation of Classification failed.')
1777             self._format_exception(e)
1778
1779     def get_classification(self, class_id):
1780         self.logger.debug(" Getting Classification %s from VIM", class_id)
1781         filter_dict = {"id": class_id}
1782         class_list = self.get_classification_list(filter_dict)
1783         if len(class_list) == 0:
1784             raise vimconn.vimconnNotFoundException(
1785                 "Classification '{}' not found".format(class_id))
1786         elif len(class_list) > 1:
1787             raise vimconn.vimconnConflictException(
1788                 "Found more than one Classification with this criteria")
1789         classification = class_list[0]
1790         return classification
1791
1792     def get_classification_list(self, filter_dict={}):
1793         self.logger.debug("Getting Classifications from VIM filter: '%s'",
1794                           str(filter_dict))
1795         try:
1796             filter_dict_os = filter_dict.copy()
1797             self._reload_connection()
1798             if self.api_version3 and "tenant_id" in filter_dict_os:
1799                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1800             classification_dict = self.neutron.list_sfc_flow_classifiers(
1801                 **filter_dict_os)
1802             classification_list = classification_dict["flow_classifiers"]
1803             self.__classification_os2mano(classification_list)
1804             return classification_list
1805         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1806                 neExceptions.NeutronException, ConnectionError) as e:
1807             self._format_exception(e)
1808
1809     def delete_classification(self, class_id):
1810         self.logger.debug("Deleting Classification '%s' from VIM", class_id)
1811         try:
1812             self._reload_connection()
1813             self.neutron.delete_sfc_flow_classifier(class_id)
1814             return class_id
1815         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1816                 ksExceptions.ClientException, neExceptions.NeutronException,
1817                 ConnectionError) as e:
1818             self._format_exception(e)
1819
1820     def new_sfi(self, name, ingress_ports, egress_ports, sfc_encap=True):
1821         self.logger.debug(
1822             "Adding a new Service Function Instance to VIM, named '%s'", name)
1823         try:
1824             new_sfi = None
1825             self._reload_connection()
1826             correlation = None
1827             if sfc_encap:
1828                 correlation = 'nsh'
1829             if len(ingress_ports) != 1:
1830                 raise vimconn.vimconnNotSupportedException(
1831                     "OpenStack VIM connector can only have "
1832                     "1 ingress port per SFI")
1833             if len(egress_ports) != 1:
1834                 raise vimconn.vimconnNotSupportedException(
1835                     "OpenStack VIM connector can only have "
1836                     "1 egress port per SFI")
1837             sfi_dict = {'name': name,
1838                         'ingress': ingress_ports[0],
1839                         'egress': egress_ports[0],
1840                         'service_function_parameters': {
1841                             'correlation': correlation}}
1842             new_sfi = self.neutron.create_sfc_port_pair({'port_pair': sfi_dict})
1843             return new_sfi['port_pair']['id']
1844         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1845                 neExceptions.NeutronException, ConnectionError) as e:
1846             if new_sfi:
1847                 try:
1848                     self.neutron.delete_sfc_port_pair(
1849                         new_sfi['port_pair']['id'])
1850                 except Exception:
1851                     self.logger.error(
1852                         'Creation of Service Function Instance failed, with '
1853                         'subsequent deletion failure as well.')
1854             self._format_exception(e)
1855
1856     def get_sfi(self, sfi_id):
1857         self.logger.debug(
1858             'Getting Service Function Instance %s from VIM', sfi_id)
1859         filter_dict = {"id": sfi_id}
1860         sfi_list = self.get_sfi_list(filter_dict)
1861         if len(sfi_list) == 0:
1862             raise vimconn.vimconnNotFoundException(
1863                 "Service Function Instance '{}' not found".format(sfi_id))
1864         elif len(sfi_list) > 1:
1865             raise vimconn.vimconnConflictException(
1866                 'Found more than one Service Function Instance '
1867                 'with this criteria')
1868         sfi = sfi_list[0]
1869         return sfi
1870
1871     def get_sfi_list(self, filter_dict={}):
1872         self.logger.debug("Getting Service Function Instances from "
1873                           "VIM filter: '%s'", str(filter_dict))
1874         try:
1875             self._reload_connection()
1876             filter_dict_os = filter_dict.copy()
1877             if self.api_version3 and "tenant_id" in filter_dict_os:
1878                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1879             sfi_dict = self.neutron.list_sfc_port_pairs(**filter_dict_os)
1880             sfi_list = sfi_dict["port_pairs"]
1881             self.__sfi_os2mano(sfi_list)
1882             return sfi_list
1883         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1884                 neExceptions.NeutronException, ConnectionError) as e:
1885             self._format_exception(e)
1886
1887     def delete_sfi(self, sfi_id):
1888         self.logger.debug("Deleting Service Function Instance '%s' "
1889                           "from VIM", sfi_id)
1890         try:
1891             self._reload_connection()
1892             self.neutron.delete_sfc_port_pair(sfi_id)
1893             return sfi_id
1894         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1895                 ksExceptions.ClientException, neExceptions.NeutronException,
1896                 ConnectionError) as e:
1897             self._format_exception(e)
1898
1899     def new_sf(self, name, sfis, sfc_encap=True):
1900         self.logger.debug("Adding a new Service Function to VIM, "
1901                           "named '%s'", name)
1902         try:
1903             new_sf = None
1904             self._reload_connection()
1905             # correlation = None
1906             # if sfc_encap:
1907             #     correlation = 'nsh'
1908             for instance in sfis:
1909                 sfi = self.get_sfi(instance)
1910                 if sfi.get('sfc_encap') != sfc_encap:
1911                     raise vimconn.vimconnNotSupportedException(
1912                         "OpenStack VIM connector requires all SFIs of the "
1913                         "same SF to share the same SFC Encapsulation")
1914             sf_dict = {'name': name,
1915                        'port_pairs': sfis}
1916             new_sf = self.neutron.create_sfc_port_pair_group({
1917                 'port_pair_group': sf_dict})
1918             return new_sf['port_pair_group']['id']
1919         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1920                 neExceptions.NeutronException, ConnectionError) as e:
1921             if new_sf:
1922                 try:
1923                     self.neutron.delete_sfc_port_pair_group(
1924                         new_sf['port_pair_group']['id'])
1925                 except Exception:
1926                     self.logger.error(
1927                         'Creation of Service Function failed, with '
1928                         'subsequent deletion failure as well.')
1929             self._format_exception(e)
1930
1931     def get_sf(self, sf_id):
1932         self.logger.debug("Getting Service Function %s from VIM", sf_id)
1933         filter_dict = {"id": sf_id}
1934         sf_list = self.get_sf_list(filter_dict)
1935         if len(sf_list) == 0:
1936             raise vimconn.vimconnNotFoundException(
1937                 "Service Function '{}' not found".format(sf_id))
1938         elif len(sf_list) > 1:
1939             raise vimconn.vimconnConflictException(
1940                 "Found more than one Service Function with this criteria")
1941         sf = sf_list[0]
1942         return sf
1943
1944     def get_sf_list(self, filter_dict={}):
1945         self.logger.debug("Getting Service Function from VIM filter: '%s'",
1946                           str(filter_dict))
1947         try:
1948             self._reload_connection()
1949             filter_dict_os = filter_dict.copy()
1950             if self.api_version3 and "tenant_id" in filter_dict_os:
1951                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
1952             sf_dict = self.neutron.list_sfc_port_pair_groups(**filter_dict_os)
1953             sf_list = sf_dict["port_pair_groups"]
1954             self.__sf_os2mano(sf_list)
1955             return sf_list
1956         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1957                 neExceptions.NeutronException, ConnectionError) as e:
1958             self._format_exception(e)
1959
1960     def delete_sf(self, sf_id):
1961         self.logger.debug("Deleting Service Function '%s' from VIM", sf_id)
1962         try:
1963             self._reload_connection()
1964             self.neutron.delete_sfc_port_pair_group(sf_id)
1965             return sf_id
1966         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
1967                 ksExceptions.ClientException, neExceptions.NeutronException,
1968                 ConnectionError) as e:
1969             self._format_exception(e)
1970
1971     def new_sfp(self, name, classifications, sfs, sfc_encap=True, spi=None):
1972         self.logger.debug("Adding a new Service Function Path to VIM, "
1973                           "named '%s'", name)
1974         try:
1975             new_sfp = None
1976             self._reload_connection()
1977             # In networking-sfc the MPLS encapsulation is legacy
1978             # should be used when no full SFC Encapsulation is intended
1979             correlation = 'mpls'
1980             if sfc_encap:
1981                 correlation = 'nsh'
1982             sfp_dict = {'name': name,
1983                         'flow_classifiers': classifications,
1984                         'port_pair_groups': sfs,
1985                         'chain_parameters': {'correlation': correlation}}
1986             if spi:
1987                 sfp_dict['chain_id'] = spi
1988             new_sfp = self.neutron.create_sfc_port_chain({'port_chain': sfp_dict})
1989             return new_sfp["port_chain"]["id"]
1990         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
1991                 neExceptions.NeutronException, ConnectionError) as e:
1992             if new_sfp:
1993                 try:
1994                     self.neutron.delete_sfc_port_chain(new_sfp['port_chain']['id'])
1995                 except Exception:
1996                     self.logger.error(
1997                         'Creation of Service Function Path failed, with '
1998                         'subsequent deletion failure as well.')
1999             self._format_exception(e)
2000
2001     def get_sfp(self, sfp_id):
2002         self.logger.debug(" Getting Service Function Path %s from VIM", sfp_id)
2003         filter_dict = {"id": sfp_id}
2004         sfp_list = self.get_sfp_list(filter_dict)
2005         if len(sfp_list) == 0:
2006             raise vimconn.vimconnNotFoundException(
2007                 "Service Function Path '{}' not found".format(sfp_id))
2008         elif len(sfp_list) > 1:
2009             raise vimconn.vimconnConflictException(
2010                 "Found more than one Service Function Path with this criteria")
2011         sfp = sfp_list[0]
2012         return sfp
2013
2014     def get_sfp_list(self, filter_dict={}):
2015         self.logger.debug("Getting Service Function Paths from VIM filter: "
2016                           "'%s'", str(filter_dict))
2017         try:
2018             self._reload_connection()
2019             filter_dict_os = filter_dict.copy()
2020             if self.api_version3 and "tenant_id" in filter_dict_os:
2021                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
2022             sfp_dict = self.neutron.list_sfc_port_chains(**filter_dict_os)
2023             sfp_list = sfp_dict["port_chains"]
2024             self.__sfp_os2mano(sfp_list)
2025             return sfp_list
2026         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2027                 neExceptions.NeutronException, ConnectionError) as e:
2028             self._format_exception(e)
2029
2030     def delete_sfp(self, sfp_id):
2031         self.logger.debug(
2032             "Deleting Service Function Path '%s' from VIM", sfp_id)
2033         try:
2034             self._reload_connection()
2035             self.neutron.delete_sfc_port_chain(sfp_id)
2036             return sfp_id
2037         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
2038                 ksExceptions.ClientException, neExceptions.NeutronException,
2039                 ConnectionError) as e:
2040             self._format_exception(e)