vimconn_openstack.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 osconnector implements all the methods to interact with openstack using the python-client.
  26 '''
  27 __author__="Alfonso Tierno, Gerardo Garcia"
  28 __date__ ="$22-jun-2014 11:19:29$"
  29
  30 import vimconn
  31 import json
  32 import yaml
  33 import logging
  34
  35 from novaclient import client as nClient, exceptions as nvExceptions
  36 import keystoneclient.v2_0.client as ksClient
  37 import keystoneclient.exceptions as ksExceptions
  38 import glanceclient.v2.client as glClient
  39 import glanceclient.client as gl1Client
  40 import glanceclient.exc as gl1Exceptions
  41 from httplib import HTTPException
  42 from neutronclient.neutron import client as neClient
  43 from neutronclient.common import exceptions as neExceptions
  44 from requests.exceptions import ConnectionError
  45
  46 '''contain the openstack virtual machine status to openmano status'''
  47 vmStatus2manoFormat={'ACTIVE':'ACTIVE',
  48                      'PAUSED':'PAUSED',
  49                      'SUSPENDED': 'SUSPENDED',
  50                      'SHUTOFF':'INACTIVE',
  51                      'BUILD':'BUILD',
  52                      'ERROR':'ERROR','DELETED':'DELETED'
  53                      }
  54 netStatus2manoFormat={'ACTIVE':'ACTIVE','PAUSED':'PAUSED','INACTIVE':'INACTIVE','BUILD':'BUILD','ERROR':'ERROR','DELETED':'DELETED'
  55                      }
  56
  57 class vimconnector(vimconn.vimconnector):
  58     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None, log_level="DEBUG", config={}):
  59         '''using common constructor parameters. In this case
  60         'url' is the keystone authorization url,
  61         'url_admin' is not use
  62         '''
  63         vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level, config)
  64
  65         self.k_creds={}
  66         self.n_creds={}
  67         if not url:
  68             raise TypeError, 'url param can not be NoneType'
  69         self.k_creds['auth_url'] = url
  70         self.n_creds['auth_url'] = url
  71         if tenant_name:
  72             self.k_creds['tenant_name'] = tenant_name
  73             self.n_creds['project_id']  = tenant_name
  74         if tenant_id:
  75             self.k_creds['tenant_id'] = tenant_id
  76             self.n_creds['tenant_id']  = tenant_id
  77         if user:
  78             self.k_creds['username'] = user
  79             self.n_creds['username'] = user
  80         if passwd:
  81             self.k_creds['password'] = passwd
  82             self.n_creds['api_key']  = passwd
  83         self.reload_client       = True
  84         self.logger = logging.getLogger('mano.vim.openstack')
  85
  86     def __setitem__(self,index, value):
  87         '''Set individuals parameters
  88         Throw TypeError, KeyError
  89         '''
  90         if index=='tenant_id':
  91             self.reload_client=True
  92             self.tenant_id = value
  93             if value:
  94                 self.k_creds['tenant_id'] = value
  95                 self.n_creds['tenant_id']  = value
  96             else:
  97                 del self.k_creds['tenant_name']
  98                 del self.n_creds['project_id']
  99         elif index=='tenant_name':
 100             self.reload_client=True
 101             self.tenant_name = value
 102             if value:
 103                 self.k_creds['tenant_name'] = value
 104                 self.n_creds['project_id']  = value
 105             else:
 106                 del self.k_creds['tenant_name']
 107                 del self.n_creds['project_id']
 108         elif index=='user':
 109             self.reload_client=True
 110             self.user = value
 111             if value:
 112                 self.k_creds['username'] = value
 113                 self.n_creds['username'] = value
 114             else:
 115                 del self.k_creds['username']
 116                 del self.n_creds['username']
 117         elif index=='passwd':
 118             self.reload_client=True
 119             self.passwd = value
 120             if value:
 121                 self.k_creds['password'] = value
 122                 self.n_creds['api_key']  = value
 123             else:
 124                 del self.k_creds['password']
 125                 del self.n_creds['api_key']
 126         elif index=='url':
 127             self.reload_client=True
 128             self.url = value
 129             if value:
 130                 self.k_creds['auth_url'] = value
 131                 self.n_creds['auth_url'] = value
 132             else:
 133                 raise TypeError, 'url param can not be NoneType'
 134         else:
 135             vimconn.vimconnector.__setitem__(self,index, value)
 136
 137     def _reload_connection(self):
 138         '''Called before any operation, it check if credentials has changed
 139         Throw keystoneclient.apiclient.exceptions.AuthorizationFailure
 140         '''
 141         #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-)
 142         if self.reload_client:
 143             #test valid params
 144             if len(self.n_creds) <4:
 145                 raise ksExceptions.ClientException("Not enough parameters to connect to openstack")
 146             self.nova = nClient.Client(2, **self.n_creds)
 147             self.keystone = ksClient.Client(**self.k_creds)
 148             self.glance_endpoint = self.keystone.service_catalog.url_for(service_type='image', endpoint_type='publicURL')
 149             self.glance = glClient.Client(self.glance_endpoint, token=self.keystone.auth_token, **self.k_creds)  #TODO check k_creds vs n_creds
 150             self.ne_endpoint=self.keystone.service_catalog.url_for(service_type='network', endpoint_type='publicURL')
 151             self.neutron = neClient.Client('2.0', endpoint_url=self.ne_endpoint, token=self.keystone.auth_token, **self.k_creds)
 152             self.reload_client = False
 153
 154     def __net_os2mano(self, net_list_dict):
 155         '''Transform the net openstack format to mano format
 156         net_list_dict can be a list of dict or a single dict'''
 157         if type(net_list_dict) is dict:
 158             net_list_=(net_list_dict,)
 159         elif type(net_list_dict) is list:
 160             net_list_=net_list_dict
 161         else:
 162             raise TypeError("param net_list_dict must be a list or a dictionary")
 163         for net in net_list_:
 164             if net.get('provider:network_type') == "vlan":
 165                 net['type']='data'
 166             else:
 167                 net['type']='bridge'
 168
 169
 170
 171     def _format_exception(self, exception):
 172         '''Transform a keystone, nova, neutron  exception into a vimconn exception'''
 173         if isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError,
 174                                   ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed,
 175                                   neClient.exceptions.ConnectionFailed)):
 176             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 177         elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException,
 178                                     neExceptions.NeutronException, nvExceptions.BadRequest)):
 179             raise vimconn.vimconnUnexpectedResponse(type(exception).__name__ + ": " + str(exception))
 180         elif isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound)):
 181             raise vimconn.vimconnNotFoundException(type(exception).__name__ + ": " + str(exception))
 182         elif isinstance(exception, nvExceptions.Conflict):
 183             raise vimconn.vimconnConflictException(type(exception).__name__ + ": " + str(exception))
 184         else: # ()
 185             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 186
 187     def get_tenant_list(self, filter_dict={}):
 188         '''Obtain tenants of VIM
 189         filter_dict can contain the following keys:
 190             name: filter by tenant name
 191             id: filter by tenant uuid/id
 192             <other VIM specific>
 193         Returns the tenant list of dictionaries: [{'name':'<name>, 'id':'<id>, ...}, ...]
 194         '''
 195         self.logger.debug("Getting tenant from VIM filter: '%s'", str(filter_dict))
 196         try:
 197             self._reload_connection()
 198             tenant_class_list=self.keystone.tenants.findall(**filter_dict)
 199             tenant_list=[]
 200             for tenant in tenant_class_list:
 201                 tenant_list.append(tenant.to_dict())
 202             return tenant_list
 203         except (ksExceptions.ConnectionError, ksExceptions.ClientException)  as e:
 204             self._format_exception(e)
 205
 206     def new_tenant(self, tenant_name, tenant_description):
 207         '''Adds a new tenant to openstack VIM. Returns the tenant identifier'''
 208         self.logger.debug("Adding a new tenant name: %s", tenant_name)
 209         try:
 210             self._reload_connection()
 211             tenant=self.keystone.tenants.create(tenant_name, tenant_description)
 212             return tenant.id
 213         except (ksExceptions.ConnectionError, ksExceptions.ClientException)  as e:
 214             self._format_exception(e)
 215
 216     def delete_tenant(self, tenant_id):
 217         '''Delete a tenant from openstack VIM. Returns the old tenant identifier'''
 218         self.logger.debug("Deleting tenant %s from VIM", tenant_id)
 219         try:
 220             self._reload_connection()
 221             self.keystone.tenants.delete(tenant_id)
 222             return tenant_id
 223         except (ksExceptions.ConnectionError, ksExceptions.ClientException)  as e:
 224             self._format_exception(e)
 225
 226     def new_network(self,net_name,net_type, shared=False, cidr=None, vlan=None):
 227         '''Adds a tenant network to VIM. Returns the network identifier'''
 228         self.logger.debug("Adding a new network to VIM name '%s', type '%s'", net_name, net_type)
 229         try:
 230             self._reload_connection()
 231             network_dict = {'name': net_name, 'admin_state_up': True}
 232             if net_type=="data" or net_type=="ptp":
 233                 if self.config.get('dataplane_physical_net') == None:
 234                     raise vimconn.vimconnConflictException("You must provide a 'dataplane_physical_net' at config value before creating sriov network")
 235                 network_dict["provider:physical_network"] = self.config['dataplane_physical_net'] #"physnet_sriov" #TODO physical
 236                 network_dict["provider:network_type"]     = "vlan"
 237                 if vlan!=None:
 238                     network_dict["provider:network_type"] = vlan
 239             network_dict["shared"]=shared
 240             new_net=self.neutron.create_network({'network':network_dict})
 241             #print new_net
 242             #create fake subnetwork
 243             if not cidr:
 244                 cidr="192.168.111.0/24"
 245             subnet={"name":net_name+"-subnet",
 246                     "network_id": new_net["network"]["id"],
 247                     "ip_version": 4,
 248                     "cidr": cidr
 249                     }
 250             self.neutron.create_subnet({"subnet": subnet} )
 251             return new_net["network"]["id"]
 252         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException) as e:
 253             self._format_exception(e)
 254
 255     def get_network_list(self, filter_dict={}):
 256         '''Obtain tenant networks of VIM
 257         Filter_dict can be:
 258             name: network name
 259             id: network uuid
 260             shared: boolean
 261             tenant_id: tenant
 262             admin_state_up: boolean
 263             status: 'ACTIVE'
 264         Returns the network list of dictionaries
 265         '''
 266         self.logger.debug("Getting network from VIM filter: '%s'", str(filter_dict))
 267         try:
 268             self._reload_connection()
 269             net_dict=self.neutron.list_networks(**filter_dict)
 270             net_list=net_dict["networks"]
 271             self.__net_os2mano(net_list)
 272             return net_list
 273         except (neExceptions.ConnectionFailed, neClient.exceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException) as e:
 274             self._format_exception(e)
 275
 276     def get_network(self, net_id):
 277         '''Obtain details of network from VIM
 278         Returns the network information from a network id'''
 279         self.logger.debug(" Getting tenant network %s from VIM", net_id)
 280         filter_dict={"id": net_id}
 281         net_list = self.get_network_list(filter_dict)
 282         if len(net_list)==0:
 283             raise vimconn.vimconnNotFoundException("Network '{}' not found".format(net_id))
 284         elif len(net_list)>1:
 285             raise vimconn.vimconnConflictException("Found more than one network with this criteria")
 286         net = net_list[0]
 287         subnets=[]
 288         for subnet_id in net.get("subnets", () ):
 289             try:
 290                 subnet = self.neutron.show_subnet(subnet_id)
 291             except Exception as e:
 292                 self.logger.error("osconnector.get_network(): Error getting subnet %s %s" % (net_id, str(e)))
 293                 subnet = {"id": subnet_id, "fault": str(e)}
 294             subnets.append(subnet)
 295         net["subnets"] = subnets
 296         return net
 297
 298     def delete_network(self, net_id):
 299         '''Deletes a tenant network from VIM. Returns the old network identifier'''
 300         self.logger.debug("Deleting network '%s' from VIM", net_id)
 301         try:
 302             self._reload_connection()
 303             #delete VM ports attached to this networks before the network
 304             ports = self.neutron.list_ports(network_id=net_id)
 305             for p in ports['ports']:
 306                 try:
 307                     self.neutron.delete_port(p["id"])
 308                 except Exception as e:
 309                     self.logger.error("Error deleting port %s: %s", p["id"], str(e))
 310             self.neutron.delete_network(net_id)
 311             return net_id
 312         except (neExceptions.ConnectionFailed, neExceptions.NetworkNotFoundClient, neExceptions.NeutronException,
 313                 neClient.exceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException) as e:
 314             self._format_exception(e)
 315
 316     def refresh_nets_status(self, net_list):
 317         '''Get the status of the networks
 318            Params: the list of network identifiers
 319            Returns a dictionary with:
 320                 net_id:         #VIM id of this network
 321                     status:     #Mandatory. Text with one of:
 322                                 #  DELETED (not found at vim)
 323                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 324                                 #  OTHER (Vim reported other status not understood)
 325                                 #  ERROR (VIM indicates an ERROR status)
 326                                 #  ACTIVE, INACTIVE, DOWN (admin down),
 327                                 #  BUILD (on building process)
 328                                 #
 329                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 330                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 331
 332         '''
 333         net_dict={}
 334         for net_id in net_list:
 335             net = {}
 336             try:
 337                 net_vim = self.get_network(net_id)
 338                 if net_vim['status'] in netStatus2manoFormat:
 339                     net["status"] = netStatus2manoFormat[ net_vim['status'] ]
 340                 else:
 341                     net["status"] = "OTHER"
 342                     net["error_msg"] = "VIM status reported " + net_vim['status']
 343
 344                 if net['status'] == "ACIVE" and not net_vim['admin_state_up']:
 345                     net['status'] = 'DOWN'
 346                 net['vim_info']  = yaml.safe_dump(net_vim)
 347                 if net_vim.get('fault'):  #TODO
 348                     net['error_msg'] = str(net_vim['fault'])
 349             except vimconn.vimconnNotFoundException as e:
 350                 self.logger.error("Exception getting net status: %s", str(e))
 351                 net['status'] = "DELETED"
 352                 net['error_msg'] = str(e)
 353             except vimconn.vimconnException as e:
 354                 self.logger.error("Exception getting net status: %s", str(e))
 355                 net['status'] = "VIM_ERROR"
 356                 net['error_msg'] = str(e)
 357             net_dict[net_id] = net
 358         return net_dict
 359
 360     def get_flavor(self, flavor_id):
 361         '''Obtain flavor details from the  VIM. Returns the flavor dict details'''
 362         self.logger.debug("Getting flavor '%s'", flavor_id)
 363         try:
 364             self._reload_connection()
 365             flavor = self.nova.flavors.find(id=flavor_id)
 366             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 367             return flavor.to_dict()
 368         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException) as e:
 369             self._format_exception(e)
 370
 371     def new_flavor(self, flavor_data, change_name_if_used=True):
 372         '''Adds a tenant flavor to openstack VIM
 373         if change_name_if_used is True, it will change name in case of conflict, because it is not supported name repetition
 374         Returns the flavor identifier
 375         '''
 376         self.logger.debug("Adding flavor '%s'", str(flavor_data))
 377         retry=0
 378         max_retries=3
 379         name_suffix = 0
 380         name=flavor_data['name']
 381         while retry<max_retries:
 382             retry+=1
 383             try:
 384                 self._reload_connection()
 385                 if change_name_if_used:
 386                     #get used names
 387                     fl_names=[]
 388                     fl=self.nova.flavors.list()
 389                     for f in fl:
 390                         fl_names.append(f.name)
 391                     while name in fl_names:
 392                         name_suffix += 1
 393                         name = flavor_data['name']+"-" + str(name_suffix)
 394
 395                 ram = flavor_data.get('ram',64)
 396                 vcpus = flavor_data.get('vcpus',1)
 397                 numa_properties=None
 398
 399                 extended = flavor_data.get("extended")
 400                 if extended:
 401                     numas=extended.get("numas")
 402                     if numas:
 403                         numa_nodes = len(numas)
 404                         if numa_nodes > 1:
 405                             return -1, "Can not add flavor with more than one numa"
 406                         numa_properties = {"hw:numa_nodes":str(numa_nodes)}
 407                         numa_properties["hw:mem_page_size"] = "large"
 408                         numa_properties["hw:cpu_policy"] = "dedicated"
 409                         numa_properties["hw:numa_mempolicy"] = "strict"
 410                         for numa in numas:
 411                             #overwrite ram and vcpus
 412                             ram = numa['memory']*1024
 413                             if 'paired-threads' in numa:
 414                                 vcpus = numa['paired-threads']*2
 415                                 numa_properties["hw:cpu_threads_policy"] = "prefer"
 416                             elif 'cores' in numa:
 417                                 vcpus = numa['cores']
 418                                 #numa_properties["hw:cpu_threads_policy"] = "prefer"
 419                             elif 'threads' in numa:
 420                                 vcpus = numa['threads']
 421                                 numa_properties["hw:cpu_policy"] = "isolated"
 422                             for interface in numa.get("interfaces",() ):
 423                                 if interface["dedicated"]=="yes":
 424                                     raise vimconn.vimconnException("Passthrough interfaces are not supported for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable)
 425                                 #TODO, add the key 'pci_passthrough:alias"="<label at config>:<number ifaces>"' when a way to connect it is available
 426
 427                 #create flavor
 428                 new_flavor=self.nova.flavors.create(name,
 429                                 ram,
 430                                 vcpus,
 431                                 flavor_data.get('disk',1),
 432                                 is_public=flavor_data.get('is_public', True)
 433                             )
 434                 #add metadata
 435                 if numa_properties:
 436                     new_flavor.set_keys(numa_properties)
 437                 return new_flavor.id
 438             except nvExceptions.Conflict as e:
 439                 if change_name_if_used and retry < max_retries:
 440                     continue
 441                 self._format_exception(e)
 442             #except nvExceptions.BadRequest as e:
 443             except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 444                 self._format_exception(e)
 445
 446     def delete_flavor(self,flavor_id):
 447         '''Deletes a tenant flavor from openstack VIM. Returns the old flavor_id
 448         '''
 449         try:
 450             self._reload_connection()
 451             self.nova.flavors.delete(flavor_id)
 452             return flavor_id
 453         #except nvExceptions.BadRequest as e:
 454         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException) as e:
 455             self._format_exception(e)
 456
 457     def new_image(self,image_dict):
 458         '''
 459         Adds a tenant image to VIM. imge_dict is a dictionary with:
 460             name: name
 461             disk_format: qcow2, vhd, vmdk, raw (by default), ...
 462             location: path or URI
 463             public: "yes" or "no"
 464             metadata: metadata of the image
 465         Returns the image_id
 466         '''
 467         #using version 1 of glance client
 468         glancev1 = gl1Client.Client('1',self.glance_endpoint, token=self.keystone.auth_token, **self.k_creds)  #TODO check k_creds vs n_creds
 469         retry=0
 470         max_retries=3
 471         while retry<max_retries:
 472             retry+=1
 473             try:
 474                 self._reload_connection()
 475                 #determine format  http://docs.openstack.org/developer/glance/formats.html
 476                 if "disk_format" in image_dict:
 477                     disk_format=image_dict["disk_format"]
 478                 else: #autodiscover base on extention
 479                     if image_dict['location'][-6:]==".qcow2":
 480                         disk_format="qcow2"
 481                     elif image_dict['location'][-4:]==".vhd":
 482                         disk_format="vhd"
 483                     elif image_dict['location'][-5:]==".vmdk":
 484                         disk_format="vmdk"
 485                     elif image_dict['location'][-4:]==".vdi":
 486                         disk_format="vdi"
 487                     elif image_dict['location'][-4:]==".iso":
 488                         disk_format="iso"
 489                     elif image_dict['location'][-4:]==".aki":
 490                         disk_format="aki"
 491                     elif image_dict['location'][-4:]==".ari":
 492                         disk_format="ari"
 493                     elif image_dict['location'][-4:]==".ami":
 494                         disk_format="ami"
 495                     else:
 496                         disk_format="raw"
 497                 self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location'])
 498                 if image_dict['location'][0:4]=="http":
 499                     new_image = glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 500                             container_format="bare", location=image_dict['location'], disk_format=disk_format)
 501                 else: #local path
 502                     with open(image_dict['location']) as fimage:
 503                         new_image = glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 504                             container_format="bare", data=fimage, disk_format=disk_format)
 505                 #insert metadata. We cannot use 'new_image.properties.setdefault'
 506                 #because nova and glance are "INDEPENDENT" and we are using nova for reading metadata
 507                 new_image_nova=self.nova.images.find(id=new_image.id)
 508                 new_image_nova.metadata.setdefault('location',image_dict['location'])
 509                 metadata_to_load = image_dict.get('metadata')
 510                 if metadata_to_load:
 511                     for k,v in yaml.load(metadata_to_load).iteritems():
 512                         new_image_nova.metadata.setdefault(k,v)
 513                 return new_image.id
 514             except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e:
 515                 self._format_exception(e)
 516             except (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError) as e:
 517                 if retry==max_retries:
 518                     continue
 519                 self._format_exception(e)
 520             except IOError as e:  #can not open the file
 521                 raise vimconn.vimconnConnectionException(type(e).__name__ + ": " + str(e)+ " for " + image_dict['location'],
 522                                                          http_code=vimconn.HTTP_Bad_Request)
 523
 524     def delete_image(self, image_id):
 525         '''Deletes a tenant image from openstack VIM. Returns the old id
 526         '''
 527         try:
 528             self._reload_connection()
 529             self.nova.images.delete(image_id)
 530             return image_id
 531         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError) as e: #TODO remove
 532             self._format_exception(e)
 533
 534     def get_image_id_from_path(self, path):
 535         '''Get the image id from image path in the VIM database. Returns the image_id
 536         '''
 537         try:
 538             self._reload_connection()
 539             images = self.nova.images.list()
 540             for image in images:
 541                 if image.metadata.get("location")==path:
 542                     return image.id
 543             raise vimconn.vimconnNotFoundException("image with location '{}' not found".format( path))
 544         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError) as e:
 545             self._format_exception(e)
 546
 547     def new_vminstance(self,name,description,start,image_id,flavor_id,net_list):
 548         '''Adds a VM instance to VIM
 549         Params:
 550             start: indicates if VM must start or boot in pause mode. Ignored
 551             image_id,flavor_id: iamge and flavor uuid
 552             net_list: list of interfaces, each one is a dictionary with:
 553                 name:
 554                 net_id: network uuid to connect
 555                 vpci: virtual vcpi to assign, ignored because openstack lack #TODO
 556                 model: interface model, ignored #TODO
 557                 mac_address: used for  SR-IOV ifaces #TODO for other types
 558                 use: 'data', 'bridge',  'mgmt'
 559                 type: 'virtual', 'PF', 'VF', 'VFnotShared'
 560                 vim_id: filled/added by this function
 561                 #TODO ip, security groups
 562         Returns the instance identifier
 563         '''
 564         self.logger.debug("Creating VM image '%s' flavor '%s' nics='%s'",image_id, flavor_id,str(net_list))
 565         try:
 566             metadata={}
 567             net_list_vim=[]
 568             self._reload_connection()
 569             metadata_vpci={} #For a specific neutron plugin
 570             for net in net_list:
 571                 if not net.get("net_id"): #skip non connected iface
 572                     continue
 573                 if net["type"]=="virtual":
 574                     net_list_vim.append({'net-id': net["net_id"]})
 575                     if "vpci" in net:
 576                         metadata_vpci[ net["net_id"] ] = [[ net["vpci"], "" ]]
 577                 elif net["type"]=="PF":
 578                     self.logger.warn("new_vminstance: Warning, can not connect a passthrough interface ")
 579                     #TODO insert this when openstack consider passthrough ports as openstack neutron ports
 580                 else: #VF
 581                     if "vpci" in net:
 582                         if "VF" not in metadata_vpci:
 583                             metadata_vpci["VF"]=[]
 584                         metadata_vpci["VF"].append([ net["vpci"], "" ])
 585                     port_dict={
 586                          "network_id": net["net_id"],
 587                          "name": net.get("name"),
 588                          "binding:vnic_type": "direct",
 589                          "admin_state_up": True
 590                     }
 591                     if not port_dict["name"]:
 592                         port_dict["name"] = name
 593                     if net.get("mac_address"):
 594                         port_dict["mac_address"]=net["mac_address"]
 595                     #TODO: manage having SRIOV without vlan tag
 596                     #if net["type"] == "VFnotShared"
 597                     #    port_dict["vlan"]=0
 598                     new_port = self.neutron.create_port({"port": port_dict })
 599                     net["mac_adress"] = new_port["port"]["mac_address"]
 600                     net["vim_id"] = new_port["port"]["id"]
 601                     net["ip"] = new_port["port"].get("fixed_ips",[{}])[0].get("ip_address")
 602                     net_list_vim.append({"port-id": new_port["port"]["id"]})
 603             if metadata_vpci:
 604                 metadata = {"pci_assignement": json.dumps(metadata_vpci)}
 605                 if len(metadata["pci_assignement"]) >255:
 606                     #limit the metadata size
 607                     #metadata["pci_assignement"] = metadata["pci_assignement"][0:255]
 608                     self.logger.warn("Metadata deleted since it exceeds the expected length (255) ")
 609                     metadata = {}
 610
 611             self.logger.debug("name '%s' image_id '%s'flavor_id '%s' net_list_vim '%s' description '%s' metadata %s",
 612                               name, image_id, flavor_id, str(net_list_vim), description, str(metadata))
 613
 614             security_groups   = self.config.get('security_groups')
 615             if type(security_groups) is str:
 616                 security_groups = ( security_groups, )
 617             server = self.nova.servers.create(name, image_id, flavor_id, nics=net_list_vim, meta=metadata,
 618                                               security_groups   = security_groups,
 619                                               availability_zone = self.config.get('availability_zone'),
 620                                               key_name          = self.config.get('keypair'),
 621                                         ) #, description=description)
 622
 623
 624             #print "DONE :-)", server
 625
 626 #             #TODO   server.add_floating_ip("10.95.87.209")
 627 #             #To look for a free floating_ip
 628 #             free_floating_ip = None
 629 #             for floating_ip in self.neutron.list_floatingips().get("floatingips", () ):
 630 #                 if not floating_ip["port_id"]:
 631 #                     free_floating_ip = floating_ip["floating_ip_address"]
 632 #                     break
 633 #             if free_floating_ip:
 634 #                 server.add_floating_ip(free_floating_ip)
 635
 636
 637             return server.id
 638 #        except nvExceptions.NotFound as e:
 639 #            error_value=-vimconn.HTTP_Not_Found
 640 #            error_text= "vm instance %s not found" % vm_id
 641         except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError,
 642                 neClient.exceptions.ConnectionFailed) as e:
 643             self._format_exception(e)
 644         except TypeError as e:
 645             raise vimconn.vimconnException(type(e).__name__ + ": "+  str(e), http_code=vimconn.HTTP_Bad_Request)
 646
 647     def get_vminstance(self,vm_id):
 648         '''Returns the VM instance information from VIM'''
 649         #self.logger.debug("Getting VM from VIM")
 650         try:
 651             self._reload_connection()
 652             server = self.nova.servers.find(id=vm_id)
 653             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 654             return server.to_dict()
 655         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound) as e:
 656             self._format_exception(e)
 657
 658     def get_vminstance_console(self,vm_id, console_type="vnc"):
 659         '''
 660         Get a console for the virtual machine
 661         Params:
 662             vm_id: uuid of the VM
 663             console_type, can be:
 664                 "novnc" (by default), "xvpvnc" for VNC types,
 665                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
 666         Returns dict with the console parameters:
 667                 protocol: ssh, ftp, http, https, ...
 668                 server:   usually ip address
 669                 port:     the http, ssh, ... port
 670                 suffix:   extra text, e.g. the http path and query string
 671         '''
 672         self.logger.debug("Getting VM CONSOLE from VIM")
 673         try:
 674             self._reload_connection()
 675             server = self.nova.servers.find(id=vm_id)
 676             if console_type == None or console_type == "novnc":
 677                 console_dict = server.get_vnc_console("novnc")
 678             elif console_type == "xvpvnc":
 679                 console_dict = server.get_vnc_console(console_type)
 680             elif console_type == "rdp-html5":
 681                 console_dict = server.get_rdp_console(console_type)
 682             elif console_type == "spice-html5":
 683                 console_dict = server.get_spice_console(console_type)
 684             else:
 685                 raise vimconn.vimconnException("console type '{}' not allowed".format(console_type), http_code=vimconn.HTTP_Bad_Request)
 686
 687             console_dict1 = console_dict.get("console")
 688             if console_dict1:
 689                 console_url = console_dict1.get("url")
 690                 if console_url:
 691                     #parse console_url
 692                     protocol_index = console_url.find("//")
 693                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
 694                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
 695                     if protocol_index < 0 or port_index<0 or suffix_index<0:
 696                         return -vimconn.HTTP_Internal_Server_Error, "Unexpected response from VIM"
 697                     console_dict={"protocol": console_url[0:protocol_index],
 698                                   "server":   console_url[protocol_index+2:port_index],
 699                                   "port":     console_url[port_index:suffix_index],
 700                                   "suffix":   console_url[suffix_index+1:]
 701                                   }
 702                     protocol_index += 2
 703                     return console_dict
 704             raise vimconn.vimconnUnexpectedResponse("Unexpected response from VIM")
 705
 706         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.BadRequest) as e:
 707             self._format_exception(e)
 708
 709     def delete_vminstance(self, vm_id):
 710         '''Removes a VM instance from VIM. Returns the old identifier
 711         '''
 712         #print "osconnector: Getting VM from VIM"
 713         try:
 714             self._reload_connection()
 715             #delete VM ports attached to this networks before the virtual machine
 716             ports = self.neutron.list_ports(device_id=vm_id)
 717             for p in ports['ports']:
 718                 try:
 719                     self.neutron.delete_port(p["id"])
 720                 except Exception as e:
 721                     self.logger.error("Error deleting port: " + type(e).__name__ + ": "+  str(e))
 722             self.nova.servers.delete(vm_id)
 723             return vm_id
 724         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException) as e:
 725             self._format_exception(e)
 726         #TODO insert exception vimconn.HTTP_Unauthorized
 727         #if reaching here is because an exception
 728
 729     def refresh_vms_status(self, vm_list):
 730         '''Get the status of the virtual machines and their interfaces/ports
 731            Params: the list of VM identifiers
 732            Returns a dictionary with:
 733                 vm_id:          #VIM id of this Virtual Machine
 734                     status:     #Mandatory. Text with one of:
 735                                 #  DELETED (not found at vim)
 736                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 737                                 #  OTHER (Vim reported other status not understood)
 738                                 #  ERROR (VIM indicates an ERROR status)
 739                                 #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
 740                                 #  CREATING (on building process), ERROR
 741                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
 742                                 #
 743                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 744                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 745                     interfaces:
 746                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
 747                         mac_address:      #Text format XX:XX:XX:XX:XX:XX
 748                         vim_net_id:       #network id where this interface is connected
 749                         vim_interface_id: #interface/port VIM id
 750                         ip_address:       #null, or text with IPv4, IPv6 address
 751         '''
 752         vm_dict={}
 753         self.logger.debug("refresh_vms status: Getting tenant VM instance information from VIM")
 754         for vm_id in vm_list:
 755             vm={}
 756             try:
 757                 vm_vim = self.get_vminstance(vm_id)
 758                 if vm_vim['status'] in vmStatus2manoFormat:
 759                     vm['status']    =  vmStatus2manoFormat[ vm_vim['status'] ]
 760                 else:
 761                     vm['status']    = "OTHER"
 762                     vm['error_msg'] = "VIM status reported " + vm_vim['status']
 763                 vm['vim_info']  = yaml.safe_dump(vm_vim)
 764                 vm["interfaces"] = []
 765                 if vm_vim.get('fault'):
 766                     vm['error_msg'] = str(vm_vim['fault'])
 767                 #get interfaces
 768                 try:
 769                     self._reload_connection()
 770                     port_dict=self.neutron.list_ports(device_id=vm_id)
 771                     for port in port_dict["ports"]:
 772                         interface={}
 773                         interface['vim_info']  = yaml.safe_dump(port)
 774                         interface["mac_address"] = port.get("mac_address")
 775                         interface["vim_net_id"] = port["network_id"]
 776                         interface["vim_interface_id"] = port["id"]
 777                         ips=[]
 778                         #look for floating ip address
 779                         floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
 780                         if floating_ip_dict.get("floatingips"):
 781                             ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address") )
 782
 783                         for subnet in port["fixed_ips"]:
 784                             ips.append(subnet["ip_address"])
 785                         interface["ip_address"] = ";".join(ips)
 786                         vm["interfaces"].append(interface)
 787                 except Exception as e:
 788                     self.logger.error("Error getting vm interface information " + type(e).__name__ + ": "+  str(e))
 789             except vimconn.vimconnNotFoundException as e:
 790                 self.logger.error("Exception getting vm status: %s", str(e))
 791                 vm['status'] = "DELETED"
 792                 vm['error_msg'] = str(e)
 793             except vimconn.vimconnException as e:
 794                 self.logger.error("Exception getting vm status: %s", str(e))
 795                 vm['status'] = "VIM_ERROR"
 796                 vm['error_msg'] = str(e)
 797             vm_dict[vm_id] = vm
 798         return vm_dict
 799
 800     def action_vminstance(self, vm_id, action_dict):
 801         '''Send and action over a VM instance from VIM
 802         Returns the vm_id if the action was successfully sent to the VIM'''
 803         self.logger.debug("Action over VM '%s': %s", vm_id, str(action_dict))
 804         try:
 805             self._reload_connection()
 806             server = self.nova.servers.find(id=vm_id)
 807             if "start" in action_dict:
 808                 if action_dict["start"]=="rebuild":
 809                     server.rebuild()
 810                 else:
 811                     if server.status=="PAUSED":
 812                         server.unpause()
 813                     elif server.status=="SUSPENDED":
 814                         server.resume()
 815                     elif server.status=="SHUTOFF":
 816                         server.start()
 817             elif "pause" in action_dict:
 818                 server.pause()
 819             elif "resume" in action_dict:
 820                 server.resume()
 821             elif "shutoff" in action_dict or "shutdown" in action_dict:
 822                 server.stop()
 823             elif "forceOff" in action_dict:
 824                 server.stop() #TODO
 825             elif "terminate" in action_dict:
 826                 server.delete()
 827             elif "createImage" in action_dict:
 828                 server.create_image()
 829                 #"path":path_schema,
 830                 #"description":description_schema,
 831                 #"name":name_schema,
 832                 #"metadata":metadata_schema,
 833                 #"imageRef": id_schema,
 834                 #"disk": {"oneOf":[{"type": "null"}, {"type":"string"}] },
 835             elif "rebuild" in action_dict:
 836                 server.rebuild(server.image['id'])
 837             elif "reboot" in action_dict:
 838                 server.reboot() #reboot_type='SOFT'
 839             elif "console" in action_dict:
 840                 console_type = action_dict["console"]
 841                 if console_type == None or console_type == "novnc":
 842                     console_dict = server.get_vnc_console("novnc")
 843                 elif console_type == "xvpvnc":
 844                     console_dict = server.get_vnc_console(console_type)
 845                 elif console_type == "rdp-html5":
 846                     console_dict = server.get_rdp_console(console_type)
 847                 elif console_type == "spice-html5":
 848                     console_dict = server.get_spice_console(console_type)
 849                 else:
 850                     raise vimconn.vimconnException("console type '{}' not allowed".format(console_type),
 851                                                    http_code=vimconn.HTTP_Bad_Request)
 852                 try:
 853                     console_url = console_dict["console"]["url"]
 854                     #parse console_url
 855                     protocol_index = console_url.find("//")
 856                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
 857                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
 858                     if protocol_index < 0 or port_index<0 or suffix_index<0:
 859                         raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
 860                     console_dict2={"protocol": console_url[0:protocol_index],
 861                                   "server":   console_url[protocol_index+2 : port_index],
 862                                   "port":     int(console_url[port_index+1 : suffix_index]),
 863                                   "suffix":   console_url[suffix_index+1:]
 864                                   }
 865                     return console_dict2
 866                 except Exception as e:
 867                     raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
 868
 869             return vm_id
 870         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound) as e:
 871             self._format_exception(e)
 872         #TODO insert exception vimconn.HTTP_Unauthorized
 873
 874 #NOT USED FUNCTIONS
 875
 876     def new_external_port(self, port_data):
 877         #TODO openstack if needed
 878         '''Adds a external port to VIM'''
 879         '''Returns the port identifier'''
 880         return -vimconn.HTTP_Internal_Server_Error, "osconnector.new_external_port() not implemented"
 881
 882     def connect_port_network(self, port_id, network_id, admin=False):
 883         #TODO openstack if needed
 884         '''Connects a external port to a network'''
 885         '''Returns status code of the VIM response'''
 886         return -vimconn.HTTP_Internal_Server_Error, "osconnector.connect_port_network() not implemented"
 887
 888     def new_user(self, user_name, user_passwd, tenant_id=None):
 889         '''Adds a new user to openstack VIM'''
 890         '''Returns the user identifier'''
 891         self.logger.debug("osconnector: Adding a new user to VIM")
 892         try:
 893             self._reload_connection()
 894             user=self.keystone.users.create(user_name, user_passwd, tenant_id=tenant_id)
 895             #self.keystone.tenants.add_user(self.k_creds["username"], #role)
 896             return user.id
 897         except ksExceptions.ConnectionError as e:
 898             error_value=-vimconn.HTTP_Bad_Request
 899             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
 900         except ksExceptions.ClientException as e: #TODO remove
 901             error_value=-vimconn.HTTP_Bad_Request
 902             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
 903         #TODO insert exception vimconn.HTTP_Unauthorized
 904         #if reaching here is because an exception
 905         if self.debug:
 906             self.logger.debug("new_user " + error_text)
 907         return error_value, error_text
 908
 909     def delete_user(self, user_id):
 910         '''Delete a user from openstack VIM'''
 911         '''Returns the user identifier'''
 912         if self.debug:
 913             print "osconnector: Deleting  a  user from VIM"
 914         try:
 915             self._reload_connection()
 916             self.keystone.users.delete(user_id)
 917             return 1, user_id
 918         except ksExceptions.ConnectionError as e:
 919             error_value=-vimconn.HTTP_Bad_Request
 920             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
 921         except ksExceptions.NotFound as e:
 922             error_value=-vimconn.HTTP_Not_Found
 923             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
 924         except ksExceptions.ClientException as e: #TODO remove
 925             error_value=-vimconn.HTTP_Bad_Request
 926             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
 927         #TODO insert exception vimconn.HTTP_Unauthorized
 928         #if reaching here is because an exception
 929         if self.debug:
 930             print "delete_tenant " + error_text
 931         return error_value, error_text
 932
 933     def get_hosts_info(self):
 934         '''Get the information of deployed hosts
 935         Returns the hosts content'''
 936         if self.debug:
 937             print "osconnector: Getting Host info from VIM"
 938         try:
 939             h_list=[]
 940             self._reload_connection()
 941             hypervisors = self.nova.hypervisors.list()
 942             for hype in hypervisors:
 943                 h_list.append( hype.to_dict() )
 944             return 1, {"hosts":h_list}
 945         except nvExceptions.NotFound as e:
 946             error_value=-vimconn.HTTP_Not_Found
 947             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
 948         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
 949             error_value=-vimconn.HTTP_Bad_Request
 950             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
 951         #TODO insert exception vimconn.HTTP_Unauthorized
 952         #if reaching here is because an exception
 953         if self.debug:
 954             print "get_hosts_info " + error_text
 955         return error_value, error_text
 956
 957     def get_hosts(self, vim_tenant):
 958         '''Get the hosts and deployed instances
 959         Returns the hosts content'''
 960         r, hype_dict = self.get_hosts_info()
 961         if r<0:
 962             return r, hype_dict
 963         hypervisors = hype_dict["hosts"]
 964         try:
 965             servers = self.nova.servers.list()
 966             for hype in hypervisors:
 967                 for server in servers:
 968                     if server.to_dict()['OS-EXT-SRV-ATTR:hypervisor_hostname']==hype['hypervisor_hostname']:
 969                         if 'vm' in hype:
 970                             hype['vm'].append(server.id)
 971                         else:
 972                             hype['vm'] = [server.id]
 973             return 1, hype_dict
 974         except nvExceptions.NotFound as e:
 975             error_value=-vimconn.HTTP_Not_Found
 976             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
 977         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
 978             error_value=-vimconn.HTTP_Bad_Request
 979             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
 980         #TODO insert exception vimconn.HTTP_Unauthorized
 981         #if reaching here is because an exception
 982         if self.debug:
 983             print "get_hosts " + error_text
 984         return error_value, error_text
 985
 986