osm_ro/vimconn_openstack.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 osconnector implements all the methods to interact with openstack using the python-client.
  26 '''
  27 __author__="Alfonso Tierno, Gerardo Garcia, Pablo Montes, xFlow Research"
  28 __date__ ="$22-jun-2014 11:19:29$"
  29
  30 import vimconn
  31 import json
  32 import yaml
  33 import logging
  34 import netaddr
  35 import time
  36 import yaml
  37 import random
  38
  39 from novaclient import client as nClient, exceptions as nvExceptions
  40 from keystoneauth1.identity import v2, v3
  41 from keystoneauth1 import session
  42 import keystoneclient.exceptions as ksExceptions
  43 from glanceclient import client as glClient
  44 import glanceclient.client as gl1Client
  45 import glanceclient.exc as gl1Exceptions
  46 from  cinderclient import client as cClient
  47 from httplib import HTTPException
  48 from neutronclient.neutron import client as neClient
  49 from neutronclient.common import exceptions as neExceptions
  50 from requests.exceptions import ConnectionError
  51
  52 '''contain the openstack virtual machine status to openmano status'''
  53 vmStatus2manoFormat={'ACTIVE':'ACTIVE',
  54                      'PAUSED':'PAUSED',
  55                      'SUSPENDED': 'SUSPENDED',
  56                      'SHUTOFF':'INACTIVE',
  57                      'BUILD':'BUILD',
  58                      'ERROR':'ERROR','DELETED':'DELETED'
  59                      }
  60 netStatus2manoFormat={'ACTIVE':'ACTIVE','PAUSED':'PAUSED','INACTIVE':'INACTIVE','BUILD':'BUILD','ERROR':'ERROR','DELETED':'DELETED'
  61                      }
  62
  63 #global var to have a timeout creating and deleting volumes
  64 volume_timeout = 60
  65 server_timeout = 60
  66
  67 class vimconnector(vimconn.vimconnector):
  68     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None,
  69                  log_level=None, config={}, persistent_info={}):
  70         '''using common constructor parameters. In this case
  71         'url' is the keystone authorization url,
  72         'url_admin' is not use
  73         '''
  74         self.osc_api_version = config.get('APIversion')
  75         if self.osc_api_version != 'v3.3' and self.osc_api_version != 'v2.0' and self.osc_api_version:
  76             raise vimconn.vimconnException("Invalid value '{}' for config:APIversion. "
  77                 "Allowed values are 'v3.3' or 'v2.0'".format(self.osc_api_version))
  78         vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level,
  79                                       config)
  80
  81         self.insecure = self.config.get("insecure", False)
  82         if not url:
  83             raise TypeError, 'url param can not be NoneType'
  84         self.auth_url = url
  85         self.tenant_name = tenant_name
  86         self.tenant_id = tenant_id
  87         self.user = user
  88         self.passwd = passwd
  89         self.persistent_info = persistent_info
  90         self.session = persistent_info.get('session', {'reload_client': True})
  91         self.nova = self.session.get('nova')
  92         self.neutron = self.session.get('neutron')
  93         self.cinder = self.session.get('cinder')
  94         self.glance = self.session.get('glance')
  95
  96         self.logger = logging.getLogger('openmano.vim.openstack')
  97         if log_level:
  98             self.logger.setLevel( getattr(logging, log_level) )
  99
 100     def __setitem__(self,index, value):
 101         '''Set individuals parameters
 102         Throw TypeError, KeyError
 103         '''
 104         self.session['reload_client'] = True
 105         vimconn.vimconnector.__setitem__(self,index, value)
 106
 107     def _reload_connection(self):
 108         '''Called before any operation, it check if credentials has changed
 109         Throw keystoneclient.apiclient.exceptions.AuthorizationFailure
 110         '''
 111         #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-)
 112         if self.session['reload_client']:
 113             if self.osc_api_version == 'v3.3' or self.osc_api_version == '3' or \
 114                     (not self.osc_api_version and self.auth_url.split("/")[-1] == "v3"):
 115                 auth = v3.Password(auth_url=self.auth_url,
 116                                    username=self.user,
 117                                    password=self.passwd,
 118                                    project_name=self.tenant_name,
 119                                    project_id=self.tenant_id,
 120                                    project_domain_id=self.config.get('project_domain_id', 'default'),
 121                                    user_domain_id=self.config.get('user_domain_id', 'default'))
 122             else:
 123                 auth = v2.Password(auth_url=self.auth_url,
 124                                    username=self.user,
 125                                    password=self.passwd,
 126                                    tenant_name=self.tenant_name,
 127                                    tenant_id=self.tenant_id)
 128             sess = session.Session(auth=auth, verify=not self.insecure)
 129             self.nova = self.session['nova'] = nClient.Client("2.1", session=sess)
 130             self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess)
 131             self.cinder = self.session['cinder'] = cClient.Client(2, session=sess)
 132             self.glance = self.session['glance'] = glClient.Client(2, session=sess)
 133             self.session['reload_client'] = False
 134             self.persistent_info['session'] = self.session
 135
 136     def __net_os2mano(self, net_list_dict):
 137         '''Transform the net openstack format to mano format
 138         net_list_dict can be a list of dict or a single dict'''
 139         if type(net_list_dict) is dict:
 140             net_list_=(net_list_dict,)
 141         elif type(net_list_dict) is list:
 142             net_list_=net_list_dict
 143         else:
 144             raise TypeError("param net_list_dict must be a list or a dictionary")
 145         for net in net_list_:
 146             if net.get('provider:network_type') == "vlan":
 147                 net['type']='data'
 148             else:
 149                 net['type']='bridge'
 150
 151
 152
 153     def _format_exception(self, exception):
 154         '''Transform a keystone, nova, neutron  exception into a vimconn exception'''
 155         if isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError,
 156                                   ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed
 157                                   )):
 158             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 159         elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException,
 160                                     neExceptions.NeutronException, nvExceptions.BadRequest)):
 161             raise vimconn.vimconnUnexpectedResponse(type(exception).__name__ + ": " + str(exception))
 162         elif isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound)):
 163             raise vimconn.vimconnNotFoundException(type(exception).__name__ + ": " + str(exception))
 164         elif isinstance(exception, nvExceptions.Conflict):
 165             raise vimconn.vimconnConflictException(type(exception).__name__ + ": " + str(exception))
 166         else: # ()
 167             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 168
 169     def get_tenant_list(self, filter_dict={}):
 170         '''Obtain tenants of VIM
 171         filter_dict can contain the following keys:
 172             name: filter by tenant name
 173             id: filter by tenant uuid/id
 174             <other VIM specific>
 175         Returns the tenant list of dictionaries: [{'name':'<name>, 'id':'<id>, ...}, ...]
 176         '''
 177         self.logger.debug("Getting tenants from VIM filter: '%s'", str(filter_dict))
 178         try:
 179             self._reload_connection()
 180             if self.osc_api_version == 'v3.3':
 181                 project_class_list=self.keystone.projects.findall(**filter_dict)
 182             else:
 183                 project_class_list=self.keystone.tenants.findall(**filter_dict)
 184             project_list=[]
 185             for project in project_class_list:
 186                 project_list.append(project.to_dict())
 187             return project_list
 188         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 189             self._format_exception(e)
 190
 191     def new_tenant(self, tenant_name, tenant_description):
 192         '''Adds a new tenant to openstack VIM. Returns the tenant identifier'''
 193         self.logger.debug("Adding a new tenant name: %s", tenant_name)
 194         try:
 195             self._reload_connection()
 196             if self.osc_api_version == 'v3.3':
 197                 project=self.keystone.projects.create(tenant_name, tenant_description)
 198             else:
 199                 project=self.keystone.tenants.create(tenant_name, tenant_description)
 200             return project.id
 201         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 202             self._format_exception(e)
 203
 204     def delete_tenant(self, tenant_id):
 205         '''Delete a tenant from openstack VIM. Returns the old tenant identifier'''
 206         self.logger.debug("Deleting tenant %s from VIM", tenant_id)
 207         try:
 208             self._reload_connection()
 209             if self.osc_api_version == 'v3.3':
 210                 self.keystone.projects.delete(tenant_id)
 211             else:
 212                 self.keystone.tenants.delete(tenant_id)
 213             return tenant_id
 214         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 215             self._format_exception(e)
 216
 217     def new_network(self,net_name, net_type, ip_profile=None, shared=False, vlan=None):
 218         '''Adds a tenant network to VIM. Returns the network identifier'''
 219         self.logger.debug("Adding a new network to VIM name '%s', type '%s'", net_name, net_type)
 220         #self.logger.debug(">>>>>>>>>>>>>>>>>> IP profile %s", str(ip_profile))
 221         try:
 222             new_net = None
 223             self._reload_connection()
 224             network_dict = {'name': net_name, 'admin_state_up': True}
 225             if net_type=="data" or net_type=="ptp":
 226                 if self.config.get('dataplane_physical_net') == None:
 227                     raise vimconn.vimconnConflictException("You must provide a 'dataplane_physical_net' at config value before creating sriov network")
 228                 network_dict["provider:physical_network"] = self.config['dataplane_physical_net'] #"physnet_sriov" #TODO physical
 229                 network_dict["provider:network_type"]     = "vlan"
 230                 if vlan!=None:
 231                     network_dict["provider:network_type"] = vlan
 232             network_dict["shared"]=shared
 233             new_net=self.neutron.create_network({'network':network_dict})
 234             #print new_net
 235             #create subnetwork, even if there is no profile
 236             if not ip_profile:
 237                 ip_profile = {}
 238             if 'subnet_address' not in ip_profile:
 239                 #Fake subnet is required
 240                 subnet_rand = random.randint(0, 255)
 241                 ip_profile['subnet_address'] = "192.168.{}.0/24".format(subnet_rand)
 242             if 'ip_version' not in ip_profile:
 243                 ip_profile['ip_version'] = "IPv4"
 244             subnet={"name":net_name+"-subnet",
 245                     "network_id": new_net["network"]["id"],
 246                     "ip_version": 4 if ip_profile['ip_version']=="IPv4" else 6,
 247                     "cidr": ip_profile['subnet_address']
 248                     }
 249             if 'gateway_address' in ip_profile:
 250                 subnet['gateway_ip'] = ip_profile['gateway_address']
 251             if ip_profile.get('dns_address'):
 252                 subnet['dns_nameservers'] = ip_profile['dns_address'].split(";")
 253             if 'dhcp_enabled' in ip_profile:
 254                 subnet['enable_dhcp'] = False if ip_profile['dhcp_enabled']=="false" else True
 255             if 'dhcp_start_address' in ip_profile:
 256                 subnet['allocation_pools']=[]
 257                 subnet['allocation_pools'].append(dict())
 258                 subnet['allocation_pools'][0]['start'] = ip_profile['dhcp_start_address']
 259             if 'dhcp_count' in ip_profile:
 260                 #parts = ip_profile['dhcp_start_address'].split('.')
 261                 #ip_int = (int(parts[0]) << 24) + (int(parts[1]) << 16) + (int(parts[2]) << 8) + int(parts[3])
 262                 ip_int = int(netaddr.IPAddress(ip_profile['dhcp_start_address']))
 263                 ip_int += ip_profile['dhcp_count'] - 1
 264                 ip_str = str(netaddr.IPAddress(ip_int))
 265                 subnet['allocation_pools'][0]['end'] = ip_str
 266             #self.logger.debug(">>>>>>>>>>>>>>>>>> Subnet: %s", str(subnet))
 267             self.neutron.create_subnet({"subnet": subnet} )
 268             return new_net["network"]["id"]
 269         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 270             if new_net:
 271                 self.neutron.delete_network(new_net['network']['id'])
 272             self._format_exception(e)
 273
 274     def get_network_list(self, filter_dict={}):
 275         '''Obtain tenant networks of VIM
 276         Filter_dict can be:
 277             name: network name
 278             id: network uuid
 279             shared: boolean
 280             tenant_id: tenant
 281             admin_state_up: boolean
 282             status: 'ACTIVE'
 283         Returns the network list of dictionaries
 284         '''
 285         self.logger.debug("Getting network from VIM filter: '%s'", str(filter_dict))
 286         try:
 287             self._reload_connection()
 288             if self.osc_api_version == 'v3.3' and "tenant_id" in filter_dict:
 289                 filter_dict['project_id'] = filter_dict.pop('tenant_id')
 290             net_dict=self.neutron.list_networks(**filter_dict)
 291             net_list=net_dict["networks"]
 292             self.__net_os2mano(net_list)
 293             return net_list
 294         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 295             self._format_exception(e)
 296
 297     def get_network(self, net_id):
 298         '''Obtain details of network from VIM
 299         Returns the network information from a network id'''
 300         self.logger.debug(" Getting tenant network %s from VIM", net_id)
 301         filter_dict={"id": net_id}
 302         net_list = self.get_network_list(filter_dict)
 303         if len(net_list)==0:
 304             raise vimconn.vimconnNotFoundException("Network '{}' not found".format(net_id))
 305         elif len(net_list)>1:
 306             raise vimconn.vimconnConflictException("Found more than one network with this criteria")
 307         net = net_list[0]
 308         subnets=[]
 309         for subnet_id in net.get("subnets", () ):
 310             try:
 311                 subnet = self.neutron.show_subnet(subnet_id)
 312             except Exception as e:
 313                 self.logger.error("osconnector.get_network(): Error getting subnet %s %s" % (net_id, str(e)))
 314                 subnet = {"id": subnet_id, "fault": str(e)}
 315             subnets.append(subnet)
 316         net["subnets"] = subnets
 317         net["encapsulation"] = net.get('provider:network_type')
 318         net["segmentation_id"] = net.get('provider:segmentation_id')
 319         return net
 320
 321     def delete_network(self, net_id):
 322         '''Deletes a tenant network from VIM. Returns the old network identifier'''
 323         self.logger.debug("Deleting network '%s' from VIM", net_id)
 324         try:
 325             self._reload_connection()
 326             #delete VM ports attached to this networks before the network
 327             ports = self.neutron.list_ports(network_id=net_id)
 328             for p in ports['ports']:
 329                 try:
 330                     self.neutron.delete_port(p["id"])
 331                 except Exception as e:
 332                     self.logger.error("Error deleting port %s: %s", p["id"], str(e))
 333             self.neutron.delete_network(net_id)
 334             return net_id
 335         except (neExceptions.ConnectionFailed, neExceptions.NetworkNotFoundClient, neExceptions.NeutronException,
 336                 ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 337             self._format_exception(e)
 338
 339     def refresh_nets_status(self, net_list):
 340         '''Get the status of the networks
 341            Params: the list of network identifiers
 342            Returns a dictionary with:
 343                 net_id:         #VIM id of this network
 344                     status:     #Mandatory. Text with one of:
 345                                 #  DELETED (not found at vim)
 346                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 347                                 #  OTHER (Vim reported other status not understood)
 348                                 #  ERROR (VIM indicates an ERROR status)
 349                                 #  ACTIVE, INACTIVE, DOWN (admin down),
 350                                 #  BUILD (on building process)
 351                                 #
 352                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 353                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 354
 355         '''
 356         net_dict={}
 357         for net_id in net_list:
 358             net = {}
 359             try:
 360                 net_vim = self.get_network(net_id)
 361                 if net_vim['status'] in netStatus2manoFormat:
 362                     net["status"] = netStatus2manoFormat[ net_vim['status'] ]
 363                 else:
 364                     net["status"] = "OTHER"
 365                     net["error_msg"] = "VIM status reported " + net_vim['status']
 366
 367                 if net['status'] == "ACTIVE" and not net_vim['admin_state_up']:
 368                     net['status'] = 'DOWN'
 369                 try:
 370                     net['vim_info'] = yaml.safe_dump(net_vim, default_flow_style=True, width=256)
 371                 except yaml.representer.RepresenterError:
 372                     net['vim_info'] = str(net_vim)
 373                 if net_vim.get('fault'):  #TODO
 374                     net['error_msg'] = str(net_vim['fault'])
 375             except vimconn.vimconnNotFoundException as e:
 376                 self.logger.error("Exception getting net status: %s", str(e))
 377                 net['status'] = "DELETED"
 378                 net['error_msg'] = str(e)
 379             except vimconn.vimconnException as e:
 380                 self.logger.error("Exception getting net status: %s", str(e))
 381                 net['status'] = "VIM_ERROR"
 382                 net['error_msg'] = str(e)
 383             net_dict[net_id] = net
 384         return net_dict
 385
 386     def get_flavor(self, flavor_id):
 387         '''Obtain flavor details from the  VIM. Returns the flavor dict details'''
 388         self.logger.debug("Getting flavor '%s'", flavor_id)
 389         try:
 390             self._reload_connection()
 391             flavor = self.nova.flavors.find(id=flavor_id)
 392             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 393             return flavor.to_dict()
 394         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 395             self._format_exception(e)
 396
 397     def get_flavor_id_from_data(self, flavor_dict):
 398         """Obtain flavor id that match the flavor description
 399            Returns the flavor_id or raises a vimconnNotFoundException
 400            flavor_dict: contains the required ram, vcpus, disk
 401            If 'use_existing_flavors' is set to True at config, the closer flavor that provides same or more ram, vcpus
 402                 and disk is returned. Otherwise a flavor with exactly same ram, vcpus and disk is returned or a
 403                 vimconnNotFoundException is raised
 404         """
 405         exact_match = False if self.config.get('use_existing_flavors') else True
 406         try:
 407             self._reload_connection()
 408             flavor_candidate_id = None
 409             flavor_candidate_data = (10000, 10000, 10000)
 410             flavor_target = (flavor_dict["ram"], flavor_dict["vcpus"], flavor_dict["disk"])
 411             # numa=None
 412             numas = flavor_dict.get("extended", {}).get("numas")
 413             if numas:
 414                 #TODO
 415                 raise vimconn.vimconnNotFoundException("Flavor with EPA still not implemted")
 416                 # if len(numas) > 1:
 417                 #     raise vimconn.vimconnNotFoundException("Cannot find any flavor with more than one numa")
 418                 # numa=numas[0]
 419                 # numas = extended.get("numas")
 420             for flavor in self.nova.flavors.list():
 421                 epa = flavor.get_keys()
 422                 if epa:
 423                     continue
 424                     # TODO
 425                 flavor_data = (flavor.ram, flavor.vcpus, flavor.disk)
 426                 if flavor_data == flavor_target:
 427                     return flavor.id
 428                 elif not exact_match and flavor_target < flavor_data < flavor_candidate_data:
 429                     flavor_candidate_id = flavor.id
 430                     flavor_candidate_data = flavor_data
 431             if not exact_match and flavor_candidate_id:
 432                 return flavor_candidate_id
 433             raise vimconn.vimconnNotFoundException("Cannot find any flavor matching '{}'".format(str(flavor_dict)))
 434         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 435             self._format_exception(e)
 436
 437
 438     def new_flavor(self, flavor_data, change_name_if_used=True):
 439         '''Adds a tenant flavor to openstack VIM
 440         if change_name_if_used is True, it will change name in case of conflict, because it is not supported name repetition
 441         Returns the flavor identifier
 442         '''
 443         self.logger.debug("Adding flavor '%s'", str(flavor_data))
 444         retry=0
 445         max_retries=3
 446         name_suffix = 0
 447         name=flavor_data['name']
 448         while retry<max_retries:
 449             retry+=1
 450             try:
 451                 self._reload_connection()
 452                 if change_name_if_used:
 453                     #get used names
 454                     fl_names=[]
 455                     fl=self.nova.flavors.list()
 456                     for f in fl:
 457                         fl_names.append(f.name)
 458                     while name in fl_names:
 459                         name_suffix += 1
 460                         name = flavor_data['name']+"-" + str(name_suffix)
 461
 462                 ram = flavor_data.get('ram',64)
 463                 vcpus = flavor_data.get('vcpus',1)
 464                 numa_properties=None
 465
 466                 extended = flavor_data.get("extended")
 467                 if extended:
 468                     numas=extended.get("numas")
 469                     if numas:
 470                         numa_nodes = len(numas)
 471                         if numa_nodes > 1:
 472                             return -1, "Can not add flavor with more than one numa"
 473                         numa_properties = {"hw:numa_nodes":str(numa_nodes)}
 474                         numa_properties["hw:mem_page_size"] = "large"
 475                         numa_properties["hw:cpu_policy"] = "dedicated"
 476                         numa_properties["hw:numa_mempolicy"] = "strict"
 477                         for numa in numas:
 478                             #overwrite ram and vcpus
 479                             ram = numa['memory']*1024
 480                             #See for reference: https://specs.openstack.org/openstack/nova-specs/specs/mitaka/implemented/virt-driver-cpu-thread-pinning.html
 481                             if 'paired-threads' in numa:
 482                                 vcpus = numa['paired-threads']*2
 483                                 #cpu_thread_policy "require" implies that the compute node must have an STM architecture
 484                                 numa_properties["hw:cpu_thread_policy"] = "require"
 485                                 numa_properties["hw:cpu_policy"] = "dedicated"
 486                             elif 'cores' in numa:
 487                                 vcpus = numa['cores']
 488                                 # cpu_thread_policy "prefer" implies that the host must not have an SMT architecture, or a non-SMT architecture will be emulated
 489                                 numa_properties["hw:cpu_thread_policy"] = "isolate"
 490                                 numa_properties["hw:cpu_policy"] = "dedicated"
 491                             elif 'threads' in numa:
 492                                 vcpus = numa['threads']
 493                                 # cpu_thread_policy "prefer" implies that the host may or may not have an SMT architecture
 494                                 numa_properties["hw:cpu_thread_policy"] = "prefer"
 495                                 numa_properties["hw:cpu_policy"] = "dedicated"
 496                             # for interface in numa.get("interfaces",() ):
 497                             #     if interface["dedicated"]=="yes":
 498                             #         raise vimconn.vimconnException("Passthrough interfaces are not supported for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable)
 499                             #     #TODO, add the key 'pci_passthrough:alias"="<label at config>:<number ifaces>"' when a way to connect it is available
 500
 501                 #create flavor
 502                 new_flavor=self.nova.flavors.create(name,
 503                                 ram,
 504                                 vcpus,
 505                                 flavor_data.get('disk',1),
 506                                 is_public=flavor_data.get('is_public', True)
 507                             )
 508                 #add metadata
 509                 if numa_properties:
 510                     new_flavor.set_keys(numa_properties)
 511                 return new_flavor.id
 512             except nvExceptions.Conflict as e:
 513                 if change_name_if_used and retry < max_retries:
 514                     continue
 515                 self._format_exception(e)
 516             #except nvExceptions.BadRequest as e:
 517             except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 518                 self._format_exception(e)
 519
 520     def delete_flavor(self,flavor_id):
 521         '''Deletes a tenant flavor from openstack VIM. Returns the old flavor_id
 522         '''
 523         try:
 524             self._reload_connection()
 525             self.nova.flavors.delete(flavor_id)
 526             return flavor_id
 527         #except nvExceptions.BadRequest as e:
 528         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 529             self._format_exception(e)
 530
 531     def new_image(self,image_dict):
 532         '''
 533         Adds a tenant image to VIM. imge_dict is a dictionary with:
 534             name: name
 535             disk_format: qcow2, vhd, vmdk, raw (by default), ...
 536             location: path or URI
 537             public: "yes" or "no"
 538             metadata: metadata of the image
 539         Returns the image_id
 540         '''
 541         # ALF TODO: revise and change for the new method or session
 542         #using version 1 of glance client
 543         glancev1 = gl1Client.Client('1',self.glance_endpoint, token=self.keystone.auth_token, **self.k_creds)  #TODO check k_creds vs n_creds
 544         retry=0
 545         max_retries=3
 546         while retry<max_retries:
 547             retry+=1
 548             try:
 549                 self._reload_connection()
 550                 #determine format  http://docs.openstack.org/developer/glance/formats.html
 551                 if "disk_format" in image_dict:
 552                     disk_format=image_dict["disk_format"]
 553                 else: #autodiscover based on extension
 554                     if image_dict['location'][-6:]==".qcow2":
 555                         disk_format="qcow2"
 556                     elif image_dict['location'][-4:]==".vhd":
 557                         disk_format="vhd"
 558                     elif image_dict['location'][-5:]==".vmdk":
 559                         disk_format="vmdk"
 560                     elif image_dict['location'][-4:]==".vdi":
 561                         disk_format="vdi"
 562                     elif image_dict['location'][-4:]==".iso":
 563                         disk_format="iso"
 564                     elif image_dict['location'][-4:]==".aki":
 565                         disk_format="aki"
 566                     elif image_dict['location'][-4:]==".ari":
 567                         disk_format="ari"
 568                     elif image_dict['location'][-4:]==".ami":
 569                         disk_format="ami"
 570                     else:
 571                         disk_format="raw"
 572                 self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location'])
 573                 if image_dict['location'][0:4]=="http":
 574                     new_image = glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 575                             container_format="bare", location=image_dict['location'], disk_format=disk_format)
 576                 else: #local path
 577                     with open(image_dict['location']) as fimage:
 578                         new_image = glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 579                             container_format="bare", data=fimage, disk_format=disk_format)
 580                 #insert metadata. We cannot use 'new_image.properties.setdefault'
 581                 #because nova and glance are "INDEPENDENT" and we are using nova for reading metadata
 582                 new_image_nova=self.nova.images.find(id=new_image.id)
 583                 new_image_nova.metadata.setdefault('location',image_dict['location'])
 584                 metadata_to_load = image_dict.get('metadata')
 585                 if metadata_to_load:
 586                     for k,v in yaml.load(metadata_to_load).iteritems():
 587                         new_image_nova.metadata.setdefault(k,v)
 588                 return new_image.id
 589             except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e:
 590                 self._format_exception(e)
 591             except (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 592                 if retry==max_retries:
 593                     continue
 594                 self._format_exception(e)
 595             except IOError as e:  #can not open the file
 596                 raise vimconn.vimconnConnectionException(type(e).__name__ + ": " + str(e)+ " for " + image_dict['location'],
 597                                                          http_code=vimconn.HTTP_Bad_Request)
 598
 599     def delete_image(self, image_id):
 600         '''Deletes a tenant image from openstack VIM. Returns the old id
 601         '''
 602         try:
 603             self._reload_connection()
 604             self.nova.images.delete(image_id)
 605             return image_id
 606         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e: #TODO remove
 607             self._format_exception(e)
 608
 609     def get_image_id_from_path(self, path):
 610         '''Get the image id from image path in the VIM database. Returns the image_id'''
 611         try:
 612             self._reload_connection()
 613             images = self.nova.images.list()
 614             for image in images:
 615                 if image.metadata.get("location")==path:
 616                     return image.id
 617             raise vimconn.vimconnNotFoundException("image with location '{}' not found".format( path))
 618         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 619             self._format_exception(e)
 620
 621     def get_image_list(self, filter_dict={}):
 622         '''Obtain tenant images from VIM
 623         Filter_dict can be:
 624             id: image id
 625             name: image name
 626             checksum: image checksum
 627         Returns the image list of dictionaries:
 628             [{<the fields at Filter_dict plus some VIM specific>}, ...]
 629             List can be empty
 630         '''
 631         self.logger.debug("Getting image list from VIM filter: '%s'", str(filter_dict))
 632         try:
 633             self._reload_connection()
 634             filter_dict_os=filter_dict.copy()
 635             #First we filter by the available filter fields: name, id. The others are removed.
 636             filter_dict_os.pop('checksum',None)
 637             image_list=self.nova.images.findall(**filter_dict_os)
 638             if len(image_list)==0:
 639                 return []
 640             #Then we filter by the rest of filter fields: checksum
 641             filtered_list = []
 642             for image in image_list:
 643                 image_class=self.glance.images.get(image.id)
 644                 if 'checksum' not in filter_dict or image_class['checksum']==filter_dict.get('checksum'):
 645                     filtered_list.append(image_class.copy())
 646             return filtered_list
 647         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 648             self._format_exception(e)
 649
 650     def new_vminstance(self,name,description,start,image_id,flavor_id,net_list,cloud_config=None,disk_list=None):
 651         '''Adds a VM instance to VIM
 652         Params:
 653             start: indicates if VM must start or boot in pause mode. Ignored
 654             image_id,flavor_id: iamge and flavor uuid
 655             net_list: list of interfaces, each one is a dictionary with:
 656                 name:
 657                 net_id: network uuid to connect
 658                 vpci: virtual vcpi to assign, ignored because openstack lack #TODO
 659                 model: interface model, ignored #TODO
 660                 mac_address: used for  SR-IOV ifaces #TODO for other types
 661                 use: 'data', 'bridge',  'mgmt'
 662                 type: 'virtual', 'PF', 'VF', 'VFnotShared'
 663                 vim_id: filled/added by this function
 664                 floating_ip: True/False (or it can be None)
 665                 #TODO ip, security groups
 666         Returns the instance identifier
 667         '''
 668         self.logger.debug("new_vminstance input: image='%s' flavor='%s' nics='%s'",image_id, flavor_id,str(net_list))
 669         try:
 670             metadata={}
 671             net_list_vim=[]
 672             external_network=[] #list of external networks to be connected to instance, later on used to create floating_ip
 673             self._reload_connection()
 674             metadata_vpci={} #For a specific neutron plugin
 675             for net in net_list:
 676                 if not net.get("net_id"): #skip non connected iface
 677                     continue
 678
 679                 port_dict={
 680                     "network_id": net["net_id"],
 681                     "name": net.get("name"),
 682                     "admin_state_up": True
 683                 }
 684                 if net["type"]=="virtual":
 685                     if "vpci" in net:
 686                         metadata_vpci[ net["net_id"] ] = [[ net["vpci"], "" ]]
 687                 elif net["type"]=="VF": # for VF
 688                     if "vpci" in net:
 689                         if "VF" not in metadata_vpci:
 690                             metadata_vpci["VF"]=[]
 691                         metadata_vpci["VF"].append([ net["vpci"], "" ])
 692                     port_dict["binding:vnic_type"]="direct"
 693                 else: #For PT
 694                     if "vpci" in net:
 695                         if "PF" not in metadata_vpci:
 696                             metadata_vpci["PF"]=[]
 697                         metadata_vpci["PF"].append([ net["vpci"], "" ])
 698                     port_dict["binding:vnic_type"]="direct-physical"
 699                 if not port_dict["name"]:
 700                     port_dict["name"]=name
 701                 if net.get("mac_address"):
 702                     port_dict["mac_address"]=net["mac_address"]
 703                 if net.get("port_security") == False:
 704                     port_dict["port_security_enabled"]=net["port_security"]
 705                 new_port = self.neutron.create_port({"port": port_dict })
 706                 net["mac_adress"] = new_port["port"]["mac_address"]
 707                 net["vim_id"] = new_port["port"]["id"]
 708                 net["ip"] = new_port["port"].get("fixed_ips", [{}])[0].get("ip_address")
 709                 net_list_vim.append({"port-id": new_port["port"]["id"]})
 710
 711                 if net.get('floating_ip', False):
 712                     net['exit_on_floating_ip_error'] = True
 713                     external_network.append(net)
 714                 elif net['use'] == 'mgmt' and self.config.get('use_floating_ip'):
 715                     net['exit_on_floating_ip_error'] = False
 716                     external_network.append(net)
 717
 718             if metadata_vpci:
 719                 metadata = {"pci_assignement": json.dumps(metadata_vpci)}
 720                 if len(metadata["pci_assignement"]) >255:
 721                     #limit the metadata size
 722                     #metadata["pci_assignement"] = metadata["pci_assignement"][0:255]
 723                     self.logger.warn("Metadata deleted since it exceeds the expected length (255) ")
 724                     metadata = {}
 725
 726             self.logger.debug("name '%s' image_id '%s'flavor_id '%s' net_list_vim '%s' description '%s' metadata %s",
 727                               name, image_id, flavor_id, str(net_list_vim), description, str(metadata))
 728
 729             security_groups   = self.config.get('security_groups')
 730             if type(security_groups) is str:
 731                 security_groups = ( security_groups, )
 732             #cloud config
 733             userdata=None
 734             config_drive = None
 735             if isinstance(cloud_config, dict):
 736                 if cloud_config.get("user-data"):
 737                     userdata=cloud_config["user-data"]
 738                 if cloud_config.get("boot-data-drive") != None:
 739                     config_drive = cloud_config["boot-data-drive"]
 740                 if cloud_config.get("config-files") or cloud_config.get("users") or cloud_config.get("key-pairs"):
 741                     if userdata:
 742                         raise vimconn.vimconnConflictException("Cloud-config cannot contain both 'userdata' and 'config-files'/'users'/'key-pairs'")
 743                     userdata_dict={}
 744                     #default user
 745                     if cloud_config.get("key-pairs"):
 746                         userdata_dict["ssh-authorized-keys"] = cloud_config["key-pairs"]
 747                         userdata_dict["users"] = [{"default": None, "ssh-authorized-keys": cloud_config["key-pairs"] }]
 748                     if cloud_config.get("users"):
 749                         if "users" not in userdata_dict:
 750                             userdata_dict["users"] = [ "default" ]
 751                         for user in cloud_config["users"]:
 752                             user_info = {
 753                                 "name" : user["name"],
 754                                 "sudo": "ALL = (ALL)NOPASSWD:ALL"
 755                             }
 756                             if "user-info" in user:
 757                                 user_info["gecos"] = user["user-info"]
 758                             if user.get("key-pairs"):
 759                                 user_info["ssh-authorized-keys"] = user["key-pairs"]
 760                             userdata_dict["users"].append(user_info)
 761
 762                     if cloud_config.get("config-files"):
 763                         userdata_dict["write_files"] = []
 764                         for file in cloud_config["config-files"]:
 765                             file_info = {
 766                                 "path" : file["dest"],
 767                                 "content": file["content"]
 768                             }
 769                             if file.get("encoding"):
 770                                 file_info["encoding"] = file["encoding"]
 771                             if file.get("permissions"):
 772                                 file_info["permissions"] = file["permissions"]
 773                             if file.get("owner"):
 774                                 file_info["owner"] = file["owner"]
 775                             userdata_dict["write_files"].append(file_info)
 776                     userdata = "#cloud-config\n"
 777                     userdata += yaml.safe_dump(userdata_dict, indent=4, default_flow_style=False)
 778                 self.logger.debug("userdata: %s", userdata)
 779             elif isinstance(cloud_config, str):
 780                 userdata = cloud_config
 781
 782             #Create additional volumes in case these are present in disk_list
 783             block_device_mapping = None
 784             base_disk_index = ord('b')
 785             if disk_list != None:
 786                 block_device_mapping = dict()
 787                 for disk in disk_list:
 788                     if 'image_id' in disk:
 789                         volume = self.cinder.volumes.create(size = disk['size'],name = name + '_vd' +
 790                                     chr(base_disk_index), imageRef = disk['image_id'])
 791                     else:
 792                         volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
 793                                     chr(base_disk_index))
 794                     block_device_mapping['_vd' +  chr(base_disk_index)] = volume.id
 795                     base_disk_index += 1
 796
 797                 #wait until volumes are with status available
 798                 keep_waiting = True
 799                 elapsed_time = 0
 800                 while keep_waiting and elapsed_time < volume_timeout:
 801                     keep_waiting = False
 802                     for volume_id in block_device_mapping.itervalues():
 803                         if self.cinder.volumes.get(volume_id).status != 'available':
 804                             keep_waiting = True
 805                     if keep_waiting:
 806                         time.sleep(1)
 807                         elapsed_time += 1
 808
 809                 #if we exceeded the timeout rollback
 810                 if elapsed_time >= volume_timeout:
 811                     #delete the volumes we just created
 812                     for volume_id in block_device_mapping.itervalues():
 813                         self.cinder.volumes.delete(volume_id)
 814
 815                     #delete ports we just created
 816                     for net_item  in net_list_vim:
 817                         if 'port-id' in net_item:
 818                             self.neutron.delete_port(net_item['port-id'])
 819
 820                     raise vimconn.vimconnException('Timeout creating volumes for instance ' + name,
 821                                                    http_code=vimconn.HTTP_Request_Timeout)
 822
 823             self.logger.debug("nova.servers.create({}, {}, {}, nics={}, meta={}, security_groups={}," \
 824                                               "availability_zone={}, key_name={}, userdata={}, config_drive={}, " \
 825                                               "block_device_mapping={})".format(name, image_id, flavor_id, net_list_vim,
 826                                                 metadata, security_groups, self.config.get('availability_zone'),
 827                                                 self.config.get('keypair'), userdata, config_drive, block_device_mapping))
 828             server = self.nova.servers.create(name, image_id, flavor_id, nics=net_list_vim, meta=metadata,
 829                                               security_groups=security_groups,
 830                                               availability_zone=self.config.get('availability_zone'),
 831                                               key_name=self.config.get('keypair'),
 832                                               userdata=userdata,
 833                                               config_drive = config_drive,
 834                                               block_device_mapping = block_device_mapping
 835                                               )  # , description=description)
 836             #print "DONE :-)", server
 837             pool_id = None
 838             floating_ips = self.neutron.list_floatingips().get("floatingips", ())
 839             for floating_network in external_network:
 840                 try:
 841                     # wait until vm is active
 842                     elapsed_time = 0
 843                     while elapsed_time < server_timeout:
 844                         status = self.nova.servers.get(server.id).status
 845                         if status == 'ACTIVE':
 846                             break
 847                         time.sleep(1)
 848                         elapsed_time += 1
 849
 850                     #if we exceeded the timeout rollback
 851                     if elapsed_time >= server_timeout:
 852                         raise vimconn.vimconnException('Timeout creating instance ' + name,
 853                                                        http_code=vimconn.HTTP_Request_Timeout)
 854
 855                     assigned = False
 856                     while(assigned == False):
 857                         if floating_ips:
 858                             ip = floating_ips.pop(0)
 859                             if not ip.get("port_id", False) and ip.get('tenant_id') == server.tenant_id:
 860                                 free_floating_ip = ip.get("floating_ip_address")
 861                                 try:
 862                                     fix_ip = floating_network.get('ip')
 863                                     server.add_floating_ip(free_floating_ip, fix_ip)
 864                                     assigned = True
 865                                 except Exception as e:
 866                                     raise vimconn.vimconnException(type(e).__name__ + ": Cannot create floating_ip "+  str(e), http_code=vimconn.HTTP_Conflict)
 867                         else:
 868                             #Find the external network
 869                             external_nets = list()
 870                             for net in self.neutron.list_networks()['networks']:
 871                                 if net['router:external']:
 872                                         external_nets.append(net)
 873
 874                             if len(external_nets) == 0:
 875                                 raise vimconn.vimconnException("Cannot create floating_ip automatically since no external "
 876                                                                "network is present",
 877                                                                 http_code=vimconn.HTTP_Conflict)
 878                             if len(external_nets) > 1:
 879                                 raise vimconn.vimconnException("Cannot create floating_ip automatically since multiple "
 880                                                                "external networks are present",
 881                                                                http_code=vimconn.HTTP_Conflict)
 882
 883                             pool_id = external_nets[0].get('id')
 884                             param = {'floatingip': {'floating_network_id': pool_id, 'tenant_id': server.tenant_id}}
 885                             try:
 886                                 #self.logger.debug("Creating floating IP")
 887                                 new_floating_ip = self.neutron.create_floatingip(param)
 888                                 free_floating_ip = new_floating_ip['floatingip']['floating_ip_address']
 889                                 fix_ip = floating_network.get('ip')
 890                                 server.add_floating_ip(free_floating_ip, fix_ip)
 891                                 assigned=True
 892                             except Exception as e:
 893                                 raise vimconn.vimconnException(type(e).__name__ + ": Cannot assign floating_ip "+  str(e), http_code=vimconn.HTTP_Conflict)
 894                 except Exception as e:
 895                     if not floating_network['exit_on_floating_ip_error']:
 896                         self.logger.warn("Cannot create floating_ip. %s", str(e))
 897                         continue
 898                     self.delete_vminstance(server.id)
 899                     raise
 900
 901             return server.id
 902 #        except nvExceptions.NotFound as e:
 903 #            error_value=-vimconn.HTTP_Not_Found
 904 #            error_text= "vm instance %s not found" % vm_id
 905         except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 906             # delete the volumes we just created
 907             if block_device_mapping != None:
 908                 for volume_id in block_device_mapping.itervalues():
 909                     self.cinder.volumes.delete(volume_id)
 910
 911             # delete ports we just created
 912             for net_item in net_list_vim:
 913                 if 'port-id' in net_item:
 914                     self.neutron.delete_port(net_item['port-id'])
 915             self._format_exception(e)
 916         except TypeError as e:
 917             raise vimconn.vimconnException(type(e).__name__ + ": "+  str(e), http_code=vimconn.HTTP_Bad_Request)
 918
 919     def get_vminstance(self,vm_id):
 920         '''Returns the VM instance information from VIM'''
 921         #self.logger.debug("Getting VM from VIM")
 922         try:
 923             self._reload_connection()
 924             server = self.nova.servers.find(id=vm_id)
 925             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 926             return server.to_dict()
 927         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
 928             self._format_exception(e)
 929
 930     def get_vminstance_console(self,vm_id, console_type="vnc"):
 931         '''
 932         Get a console for the virtual machine
 933         Params:
 934             vm_id: uuid of the VM
 935             console_type, can be:
 936                 "novnc" (by default), "xvpvnc" for VNC types,
 937                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
 938         Returns dict with the console parameters:
 939                 protocol: ssh, ftp, http, https, ...
 940                 server:   usually ip address
 941                 port:     the http, ssh, ... port
 942                 suffix:   extra text, e.g. the http path and query string
 943         '''
 944         self.logger.debug("Getting VM CONSOLE from VIM")
 945         try:
 946             self._reload_connection()
 947             server = self.nova.servers.find(id=vm_id)
 948             if console_type == None or console_type == "novnc":
 949                 console_dict = server.get_vnc_console("novnc")
 950             elif console_type == "xvpvnc":
 951                 console_dict = server.get_vnc_console(console_type)
 952             elif console_type == "rdp-html5":
 953                 console_dict = server.get_rdp_console(console_type)
 954             elif console_type == "spice-html5":
 955                 console_dict = server.get_spice_console(console_type)
 956             else:
 957                 raise vimconn.vimconnException("console type '{}' not allowed".format(console_type), http_code=vimconn.HTTP_Bad_Request)
 958
 959             console_dict1 = console_dict.get("console")
 960             if console_dict1:
 961                 console_url = console_dict1.get("url")
 962                 if console_url:
 963                     #parse console_url
 964                     protocol_index = console_url.find("//")
 965                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
 966                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
 967                     if protocol_index < 0 or port_index<0 or suffix_index<0:
 968                         return -vimconn.HTTP_Internal_Server_Error, "Unexpected response from VIM"
 969                     console_dict={"protocol": console_url[0:protocol_index],
 970                                   "server":   console_url[protocol_index+2:port_index],
 971                                   "port":     console_url[port_index:suffix_index],
 972                                   "suffix":   console_url[suffix_index+1:]
 973                                   }
 974                     protocol_index += 2
 975                     return console_dict
 976             raise vimconn.vimconnUnexpectedResponse("Unexpected response from VIM")
 977
 978         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.BadRequest, ConnectionError) as e:
 979             self._format_exception(e)
 980
 981     def delete_vminstance(self, vm_id):
 982         '''Removes a VM instance from VIM. Returns the old identifier
 983         '''
 984         #print "osconnector: Getting VM from VIM"
 985         try:
 986             self._reload_connection()
 987             #delete VM ports attached to this networks before the virtual machine
 988             ports = self.neutron.list_ports(device_id=vm_id)
 989             for p in ports['ports']:
 990                 try:
 991                     self.neutron.delete_port(p["id"])
 992                 except Exception as e:
 993                     self.logger.error("Error deleting port: " + type(e).__name__ + ": "+  str(e))
 994
 995             #commented because detaching the volumes makes the servers.delete not work properly ?!?
 996             #dettach volumes attached
 997             server = self.nova.servers.get(vm_id)
 998             volumes_attached_dict = server._info['os-extended-volumes:volumes_attached']
 999             #for volume in volumes_attached_dict:
1000             #    self.cinder.volumes.detach(volume['id'])
1001
1002             self.nova.servers.delete(vm_id)
1003
1004             #delete volumes.
1005             #Although having detached them should have them  in active status
1006             #we ensure in this loop
1007             keep_waiting = True
1008             elapsed_time = 0
1009             while keep_waiting and elapsed_time < volume_timeout:
1010                 keep_waiting = False
1011                 for volume in volumes_attached_dict:
1012                     if self.cinder.volumes.get(volume['id']).status != 'available':
1013                         keep_waiting = True
1014                     else:
1015                         self.cinder.volumes.delete(volume['id'])
1016                 if keep_waiting:
1017                     time.sleep(1)
1018                     elapsed_time += 1
1019
1020             return vm_id
1021         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
1022             self._format_exception(e)
1023         #TODO insert exception vimconn.HTTP_Unauthorized
1024         #if reaching here is because an exception
1025
1026     def refresh_vms_status(self, vm_list):
1027         '''Get the status of the virtual machines and their interfaces/ports
1028            Params: the list of VM identifiers
1029            Returns a dictionary with:
1030                 vm_id:          #VIM id of this Virtual Machine
1031                     status:     #Mandatory. Text with one of:
1032                                 #  DELETED (not found at vim)
1033                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1034                                 #  OTHER (Vim reported other status not understood)
1035                                 #  ERROR (VIM indicates an ERROR status)
1036                                 #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
1037                                 #  CREATING (on building process), ERROR
1038                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
1039                                 #
1040                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
1041                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
1042                     interfaces:
1043                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
1044                         mac_address:      #Text format XX:XX:XX:XX:XX:XX
1045                         vim_net_id:       #network id where this interface is connected
1046                         vim_interface_id: #interface/port VIM id
1047                         ip_address:       #null, or text with IPv4, IPv6 address
1048                         compute_node:     #identification of compute node where PF,VF interface is allocated
1049                         pci:              #PCI address of the NIC that hosts the PF,VF
1050                         vlan:             #physical VLAN used for VF
1051         '''
1052         vm_dict={}
1053         self.logger.debug("refresh_vms status: Getting tenant VM instance information from VIM")
1054         for vm_id in vm_list:
1055             vm={}
1056             try:
1057                 vm_vim = self.get_vminstance(vm_id)
1058                 if vm_vim['status'] in vmStatus2manoFormat:
1059                     vm['status']    =  vmStatus2manoFormat[ vm_vim['status'] ]
1060                 else:
1061                     vm['status']    = "OTHER"
1062                     vm['error_msg'] = "VIM status reported " + vm_vim['status']
1063                 try:
1064                     vm['vim_info']  = yaml.safe_dump(vm_vim, default_flow_style=True, width=256)
1065                 except yaml.representer.RepresenterError:
1066                     vm['vim_info'] = str(vm_vim)
1067                 vm["interfaces"] = []
1068                 if vm_vim.get('fault'):
1069                     vm['error_msg'] = str(vm_vim['fault'])
1070                 #get interfaces
1071                 try:
1072                     self._reload_connection()
1073                     port_dict=self.neutron.list_ports(device_id=vm_id)
1074                     for port in port_dict["ports"]:
1075                         interface={}
1076                         try:
1077                             interface['vim_info'] = yaml.safe_dump(port, default_flow_style=True, width=256)
1078                         except yaml.representer.RepresenterError:
1079                             interface['vim_info'] = str(port)
1080                         interface["mac_address"] = port.get("mac_address")
1081                         interface["vim_net_id"] = port["network_id"]
1082                         interface["vim_interface_id"] = port["id"]
1083                         # check if OS-EXT-SRV-ATTR:host is there,
1084                         # in case of non-admin credentials, it will be missing
1085                         if vm_vim.get('OS-EXT-SRV-ATTR:host'):
1086                             interface["compute_node"] = vm_vim['OS-EXT-SRV-ATTR:host']
1087                         interface["pci"] = None
1088
1089                         # check if binding:profile is there,
1090                         # in case of non-admin credentials, it will be missing
1091                         if port.get('binding:profile'):
1092                             if port['binding:profile'].get('pci_slot'):
1093                                 # TODO: At the moment sr-iov pci addresses are converted to PF pci addresses by setting the slot to 0x00
1094                                 # TODO: This is just a workaround valid for niantinc. Find a better way to do so
1095                                 #   CHANGE DDDD:BB:SS.F to DDDD:BB:00.(F%2)   assuming there are 2 ports per nic
1096                                 pci = port['binding:profile']['pci_slot']
1097                                 # interface["pci"] = pci[:-4] + "00." + str(int(pci[-1]) % 2)
1098                                 interface["pci"] = pci
1099                         interface["vlan"] = None
1100                         #if network is of type vlan and port is of type direct (sr-iov) then set vlan id
1101                         network = self.neutron.show_network(port["network_id"])
1102                         if network['network'].get('provider:network_type') == 'vlan' and \
1103                             port.get("binding:vnic_type") == "direct":
1104                             interface["vlan"] = network['network'].get('provider:segmentation_id')
1105                         ips=[]
1106                         #look for floating ip address
1107                         floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
1108                         if floating_ip_dict.get("floatingips"):
1109                             ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address") )
1110
1111                         for subnet in port["fixed_ips"]:
1112                             ips.append(subnet["ip_address"])
1113                         interface["ip_address"] = ";".join(ips)
1114                         vm["interfaces"].append(interface)
1115                 except Exception as e:
1116                     self.logger.error("Error getting vm interface information " + type(e).__name__ + ": "+  str(e))
1117             except vimconn.vimconnNotFoundException as e:
1118                 self.logger.error("Exception getting vm status: %s", str(e))
1119                 vm['status'] = "DELETED"
1120                 vm['error_msg'] = str(e)
1121             except vimconn.vimconnException as e:
1122                 self.logger.error("Exception getting vm status: %s", str(e))
1123                 vm['status'] = "VIM_ERROR"
1124                 vm['error_msg'] = str(e)
1125             vm_dict[vm_id] = vm
1126         return vm_dict
1127
1128     def action_vminstance(self, vm_id, action_dict):
1129         '''Send and action over a VM instance from VIM
1130         Returns the vm_id if the action was successfully sent to the VIM'''
1131         self.logger.debug("Action over VM '%s': %s", vm_id, str(action_dict))
1132         try:
1133             self._reload_connection()
1134             server = self.nova.servers.find(id=vm_id)
1135             if "start" in action_dict:
1136                 if action_dict["start"]=="rebuild":
1137                     server.rebuild()
1138                 else:
1139                     if server.status=="PAUSED":
1140                         server.unpause()
1141                     elif server.status=="SUSPENDED":
1142                         server.resume()
1143                     elif server.status=="SHUTOFF":
1144                         server.start()
1145             elif "pause" in action_dict:
1146                 server.pause()
1147             elif "resume" in action_dict:
1148                 server.resume()
1149             elif "shutoff" in action_dict or "shutdown" in action_dict:
1150                 server.stop()
1151             elif "forceOff" in action_dict:
1152                 server.stop() #TODO
1153             elif "terminate" in action_dict:
1154                 server.delete()
1155             elif "createImage" in action_dict:
1156                 server.create_image()
1157                 #"path":path_schema,
1158                 #"description":description_schema,
1159                 #"name":name_schema,
1160                 #"metadata":metadata_schema,
1161                 #"imageRef": id_schema,
1162                 #"disk": {"oneOf":[{"type": "null"}, {"type":"string"}] },
1163             elif "rebuild" in action_dict:
1164                 server.rebuild(server.image['id'])
1165             elif "reboot" in action_dict:
1166                 server.reboot() #reboot_type='SOFT'
1167             elif "console" in action_dict:
1168                 console_type = action_dict["console"]
1169                 if console_type == None or console_type == "novnc":
1170                     console_dict = server.get_vnc_console("novnc")
1171                 elif console_type == "xvpvnc":
1172                     console_dict = server.get_vnc_console(console_type)
1173                 elif console_type == "rdp-html5":
1174                     console_dict = server.get_rdp_console(console_type)
1175                 elif console_type == "spice-html5":
1176                     console_dict = server.get_spice_console(console_type)
1177                 else:
1178                     raise vimconn.vimconnException("console type '{}' not allowed".format(console_type),
1179                                                    http_code=vimconn.HTTP_Bad_Request)
1180                 try:
1181                     console_url = console_dict["console"]["url"]
1182                     #parse console_url
1183                     protocol_index = console_url.find("//")
1184                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1185                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1186                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1187                         raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1188                     console_dict2={"protocol": console_url[0:protocol_index],
1189                                   "server":   console_url[protocol_index+2 : port_index],
1190                                   "port":     int(console_url[port_index+1 : suffix_index]),
1191                                   "suffix":   console_url[suffix_index+1:]
1192                                   }
1193                     return console_dict2
1194                 except Exception as e:
1195                     raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1196
1197             return vm_id
1198         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1199             self._format_exception(e)
1200         #TODO insert exception vimconn.HTTP_Unauthorized
1201
1202 #NOT USED FUNCTIONS
1203
1204     def new_external_port(self, port_data):
1205         #TODO openstack if needed
1206         '''Adds a external port to VIM'''
1207         '''Returns the port identifier'''
1208         return -vimconn.HTTP_Internal_Server_Error, "osconnector.new_external_port() not implemented"
1209
1210     def connect_port_network(self, port_id, network_id, admin=False):
1211         #TODO openstack if needed
1212         '''Connects a external port to a network'''
1213         '''Returns status code of the VIM response'''
1214         return -vimconn.HTTP_Internal_Server_Error, "osconnector.connect_port_network() not implemented"
1215
1216     def new_user(self, user_name, user_passwd, tenant_id=None):
1217         '''Adds a new user to openstack VIM'''
1218         '''Returns the user identifier'''
1219         self.logger.debug("osconnector: Adding a new user to VIM")
1220         try:
1221             self._reload_connection()
1222             user=self.keystone.users.create(user_name, user_passwd, tenant_id=tenant_id)
1223             #self.keystone.tenants.add_user(self.k_creds["username"], #role)
1224             return user.id
1225         except ksExceptions.ConnectionError as e:
1226             error_value=-vimconn.HTTP_Bad_Request
1227             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1228         except ksExceptions.ClientException as e: #TODO remove
1229             error_value=-vimconn.HTTP_Bad_Request
1230             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1231         #TODO insert exception vimconn.HTTP_Unauthorized
1232         #if reaching here is because an exception
1233         if self.debug:
1234             self.logger.debug("new_user " + error_text)
1235         return error_value, error_text
1236
1237     def delete_user(self, user_id):
1238         '''Delete a user from openstack VIM'''
1239         '''Returns the user identifier'''
1240         if self.debug:
1241             print "osconnector: Deleting  a  user from VIM"
1242         try:
1243             self._reload_connection()
1244             self.keystone.users.delete(user_id)
1245             return 1, user_id
1246         except ksExceptions.ConnectionError as e:
1247             error_value=-vimconn.HTTP_Bad_Request
1248             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1249         except ksExceptions.NotFound as e:
1250             error_value=-vimconn.HTTP_Not_Found
1251             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1252         except ksExceptions.ClientException as e: #TODO remove
1253             error_value=-vimconn.HTTP_Bad_Request
1254             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1255         #TODO insert exception vimconn.HTTP_Unauthorized
1256         #if reaching here is because an exception
1257         if self.debug:
1258             print "delete_tenant " + error_text
1259         return error_value, error_text
1260
1261     def get_hosts_info(self):
1262         '''Get the information of deployed hosts
1263         Returns the hosts content'''
1264         if self.debug:
1265             print "osconnector: Getting Host info from VIM"
1266         try:
1267             h_list=[]
1268             self._reload_connection()
1269             hypervisors = self.nova.hypervisors.list()
1270             for hype in hypervisors:
1271                 h_list.append( hype.to_dict() )
1272             return 1, {"hosts":h_list}
1273         except nvExceptions.NotFound as e:
1274             error_value=-vimconn.HTTP_Not_Found
1275             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1276         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1277             error_value=-vimconn.HTTP_Bad_Request
1278             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1279         #TODO insert exception vimconn.HTTP_Unauthorized
1280         #if reaching here is because an exception
1281         if self.debug:
1282             print "get_hosts_info " + error_text
1283         return error_value, error_text
1284
1285     def get_hosts(self, vim_tenant):
1286         '''Get the hosts and deployed instances
1287         Returns the hosts content'''
1288         r, hype_dict = self.get_hosts_info()
1289         if r<0:
1290             return r, hype_dict
1291         hypervisors = hype_dict["hosts"]
1292         try:
1293             servers = self.nova.servers.list()
1294             for hype in hypervisors:
1295                 for server in servers:
1296                     if server.to_dict()['OS-EXT-SRV-ATTR:hypervisor_hostname']==hype['hypervisor_hostname']:
1297                         if 'vm' in hype:
1298                             hype['vm'].append(server.id)
1299                         else:
1300                             hype['vm'] = [server.id]
1301             return 1, hype_dict
1302         except nvExceptions.NotFound as e:
1303             error_value=-vimconn.HTTP_Not_Found
1304             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1305         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1306             error_value=-vimconn.HTTP_Bad_Request
1307             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1308         #TODO insert exception vimconn.HTTP_Unauthorized
1309         #if reaching here is because an exception
1310         if self.debug:
1311             print "get_hosts " + error_text
1312         return error_value, error_text
1313
1314