vimconn_openstack.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 osconnector implements all the methods to interact with openstack using the python-client.
  26 '''
  27 __author__="Alfonso Tierno, Gerardo Garcia"
  28 __date__ ="$22-jun-2014 11:19:29$"
  29
  30 import vimconn
  31 import json
  32 import yaml
  33 import logging
  34
  35 from novaclient import client as nClient, exceptions as nvExceptions
  36 import keystoneclient.v2_0.client as ksClient
  37 import keystoneclient.exceptions as ksExceptions
  38 import glanceclient.v2.client as glClient
  39 import glanceclient.client as gl1Client
  40 import glanceclient.exc as gl1Exceptions
  41 from httplib import HTTPException
  42 from neutronclient.neutron import client as neClient
  43 from neutronclient.common import exceptions as neExceptions
  44 from requests.exceptions import ConnectionError
  45
  46 '''contain the openstack virtual machine status to openmano status'''
  47 vmStatus2manoFormat={'ACTIVE':'ACTIVE',
  48                      'PAUSED':'PAUSED',
  49                      'SUSPENDED': 'SUSPENDED',
  50                      'SHUTOFF':'INACTIVE',
  51                      'BUILD':'BUILD',
  52                      'ERROR':'ERROR','DELETED':'DELETED'
  53                      }
  54 netStatus2manoFormat={'ACTIVE':'ACTIVE','PAUSED':'PAUSED','INACTIVE':'INACTIVE','BUILD':'BUILD','ERROR':'ERROR','DELETED':'DELETED'
  55                      }
  56
  57 class vimconnector(vimconn.vimconnector):
  58     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None, log_level="DEBUG", config={}):
  59         '''using common constructor parameters. In this case
  60         'url' is the keystone authorization url,
  61         'url_admin' is not use
  62         '''
  63         vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level, config)
  64
  65         self.k_creds={}
  66         self.n_creds={}
  67         if not url:
  68             raise TypeError, 'url param can not be NoneType'
  69         self.k_creds['auth_url'] = url
  70         self.n_creds['auth_url'] = url
  71         if tenant_name:
  72             self.k_creds['tenant_name'] = tenant_name
  73             self.n_creds['project_id']  = tenant_name
  74         if tenant_id:
  75             self.k_creds['tenant_id'] = tenant_id
  76             self.n_creds['tenant_id']  = tenant_id
  77         if user:
  78             self.k_creds['username'] = user
  79             self.n_creds['username'] = user
  80         if passwd:
  81             self.k_creds['password'] = passwd
  82             self.n_creds['api_key']  = passwd
  83         self.reload_client       = True
  84         self.logger = logging.getLogger('mano.vim.openstack')
  85
  86     def __setitem__(self,index, value):
  87         '''Set individuals parameters
  88         Throw TypeError, KeyError
  89         '''
  90         if index=='tenant_id':
  91             self.reload_client=True
  92             self.tenant_id = value
  93             if value:
  94                 self.k_creds['tenant_id'] = value
  95                 self.n_creds['tenant_id']  = value
  96             else:
  97                 del self.k_creds['tenant_name']
  98                 del self.n_creds['project_id']
  99         elif index=='tenant_name':
 100             self.reload_client=True
 101             self.tenant_name = value
 102             if value:
 103                 self.k_creds['tenant_name'] = value
 104                 self.n_creds['project_id']  = value
 105             else:
 106                 del self.k_creds['tenant_name']
 107                 del self.n_creds['project_id']
 108         elif index=='user':
 109             self.reload_client=True
 110             self.user = value
 111             if value:
 112                 self.k_creds['username'] = value
 113                 self.n_creds['username'] = value
 114             else:
 115                 del self.k_creds['username']
 116                 del self.n_creds['username']
 117         elif index=='passwd':
 118             self.reload_client=True
 119             self.passwd = value
 120             if value:
 121                 self.k_creds['password'] = value
 122                 self.n_creds['api_key']  = value
 123             else:
 124                 del self.k_creds['password']
 125                 del self.n_creds['api_key']
 126         elif index=='url':
 127             self.reload_client=True
 128             self.url = value
 129             if value:
 130                 self.k_creds['auth_url'] = value
 131                 self.n_creds['auth_url'] = value
 132             else:
 133                 raise TypeError, 'url param can not be NoneType'
 134         else:
 135             vimconn.vimconnector.__setitem__(self,index, value)
 136
 137     def _reload_connection(self):
 138         '''Called before any operation, it check if credentials has changed
 139         Throw keystoneclient.apiclient.exceptions.AuthorizationFailure
 140         '''
 141         #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-)
 142         if self.reload_client:
 143             #test valid params
 144             if len(self.n_creds) <4:
 145                 raise ksExceptions.ClientException("Not enough parameters to connect to openstack")
 146             self.nova = nClient.Client(2, **self.n_creds)
 147             self.keystone = ksClient.Client(**self.k_creds)
 148             self.glance_endpoint = self.keystone.service_catalog.url_for(service_type='image', endpoint_type='publicURL')
 149             self.glance = glClient.Client(self.glance_endpoint, token=self.keystone.auth_token, **self.k_creds)  #TODO check k_creds vs n_creds
 150             self.ne_endpoint=self.keystone.service_catalog.url_for(service_type='network', endpoint_type='publicURL')
 151             self.neutron = neClient.Client('2.0', endpoint_url=self.ne_endpoint, token=self.keystone.auth_token, **self.k_creds)
 152             self.reload_client = False
 153
 154     def __net_os2mano(self, net_list_dict):
 155         '''Transform the net openstack format to mano format
 156         net_list_dict can be a list of dict or a single dict'''
 157         if type(net_list_dict) is dict:
 158             net_list_=(net_list_dict,)
 159         elif type(net_list_dict) is list:
 160             net_list_=net_list_dict
 161         else:
 162             raise TypeError("param net_list_dict must be a list or a dictionary")
 163         for net in net_list_:
 164             if net.get('provider:network_type') == "vlan":
 165                 net['type']='data'
 166             else:
 167                 net['type']='bridge'
 168
 169
 170
 171     def _format_exception(self, exception):
 172         '''Transform a keystone, nova, neutron  exception into a vimconn exception'''
 173         if isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError,
 174                                   ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed,
 175                                   neClient.exceptions.ConnectionFailed)):
 176             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 177         elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException,
 178                                     neExceptions.NeutronException, nvExceptions.BadRequest)):
 179             raise vimconn.vimconnUnexpectedResponse(type(exception).__name__ + ": " + str(exception))
 180         elif isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound)):
 181             raise vimconn.vimconnNotFoundException(type(exception).__name__ + ": " + str(exception))
 182         elif isinstance(exception, nvExceptions.Conflict):
 183             raise vimconn.vimconnConflictException(type(exception).__name__ + ": " + str(exception))
 184         else: # ()
 185             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 186
 187     def get_tenant_list(self, filter_dict={}):
 188         '''Obtain tenants of VIM
 189         filter_dict can contain the following keys:
 190             name: filter by tenant name
 191             id: filter by tenant uuid/id
 192             <other VIM specific>
 193         Returns the tenant list of dictionaries: [{'name':'<name>, 'id':'<id>, ...}, ...]
 194         '''
 195         self.logger.debug("Getting tenant from VIM filter: '%s'", str(filter_dict))
 196         try:
 197             self._reload_connection()
 198             tenant_class_list=self.keystone.tenants.findall(**filter_dict)
 199             tenant_list=[]
 200             for tenant in tenant_class_list:
 201                 tenant_list.append(tenant.to_dict())
 202             return tenant_list
 203         except (ksExceptions.ConnectionError, ksExceptions.ClientException)  as e:
 204             self._format_exception(e)
 205
 206     def new_tenant(self, tenant_name, tenant_description):
 207         '''Adds a new tenant to openstack VIM. Returns the tenant identifier'''
 208         self.logger.debug("Adding a new tenant name: %s", tenant_name)
 209         try:
 210             self._reload_connection()
 211             tenant=self.keystone.tenants.create(tenant_name, tenant_description)
 212             return tenant.id
 213         except (ksExceptions.ConnectionError, ksExceptions.ClientException)  as e:
 214             self._format_exception(e)
 215
 216     def delete_tenant(self, tenant_id):
 217         '''Delete a tenant from openstack VIM. Returns the old tenant identifier'''
 218         self.logger.debug("Deleting tenant %s from VIM", tenant_id)
 219         try:
 220             self._reload_connection()
 221             self.keystone.tenants.delete(tenant_id)
 222             return tenant_id
 223         except (ksExceptions.ConnectionError, ksExceptions.ClientException)  as e:
 224             self._format_exception(e)
 225
 226     def new_network(self,net_name,net_type, shared=False, cidr=None, vlan=None):
 227         '''Adds a tenant network to VIM. Returns the network identifier'''
 228         self.logger.debug("Adding a new network to VIM name '%s', type '%s'", net_name, net_type)
 229         try:
 230             self._reload_connection()
 231             network_dict = {'name': net_name, 'admin_state_up': True}
 232             if net_type=="data" or net_type=="ptp":
 233                 if self.config.get('dataplane_physical_net') == None:
 234                     raise vimconn.vimconnConflictException("You must provide a 'dataplane_physical_net' at config value before creating sriov network")
 235                 network_dict["provider:physical_network"] = self.config['dataplane_physical_net'] #"physnet_sriov" #TODO physical
 236                 network_dict["provider:network_type"]     = "vlan"
 237                 if vlan!=None:
 238                     network_dict["provider:network_type"] = vlan
 239             network_dict["shared"]=shared
 240             new_net=self.neutron.create_network({'network':network_dict})
 241             #print new_net
 242             #create fake subnetwork
 243             if not cidr:
 244                 cidr="192.168.111.0/24"
 245             subnet={"name":net_name+"-subnet",
 246                     "network_id": new_net["network"]["id"],
 247                     "ip_version": 4,
 248                     "cidr": cidr
 249                     }
 250             self.neutron.create_subnet({"subnet": subnet} )
 251             return new_net["network"]["id"]
 252         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException) as e:
 253             self._format_exception(e)
 254
 255     def get_network_list(self, filter_dict={}):
 256         '''Obtain tenant networks of VIM
 257         Filter_dict can be:
 258             name: network name
 259             id: network uuid
 260             shared: boolean
 261             tenant_id: tenant
 262             admin_state_up: boolean
 263             status: 'ACTIVE'
 264         Returns the network list of dictionaries
 265         '''
 266         self.logger.debug("Getting network from VIM filter: '%s'", str(filter_dict))
 267         try:
 268             self._reload_connection()
 269             net_dict=self.neutron.list_networks(**filter_dict)
 270             net_list=net_dict["networks"]
 271             self.__net_os2mano(net_list)
 272             return net_list
 273         except (neExceptions.ConnectionFailed, neClient.exceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException) as e:
 274             self._format_exception(e)
 275
 276     def get_network(self, net_id):
 277         '''Obtain details of network from VIM
 278         Returns the network information from a network id'''
 279         self.logger.debug(" Getting tenant network %s from VIM", net_id)
 280         filter_dict={"id": net_id}
 281         net_list = self.get_network_list(filter_dict)
 282         if len(net_list)==0:
 283             raise vimconn.vimconnNotFoundException("Network '{}' not found".format(net_id))
 284         elif len(net_list)>1:
 285             raise vimconn.vimconnConflictException("Found more than one network with this criteria")
 286         net = net_list[0]
 287         subnets=[]
 288         for subnet_id in net.get("subnets", () ):
 289             try:
 290                 subnet = self.neutron.show_subnet(subnet_id)
 291             except Exception as e:
 292                 self.logger.error("osconnector.get_network(): Error getting subnet %s %s" % (net_id, str(e)))
 293                 subnet = {"id": subnet_id, "fault": str(e)}
 294             subnets.append(subnet)
 295         net["subnets"] = subnets
 296         return net
 297
 298     def delete_network(self, net_id):
 299         '''Deletes a tenant network from VIM. Returns the old network identifier'''
 300         self.logger.debug("Deleting network '%s' from VIM", net_id)
 301         try:
 302             self._reload_connection()
 303             #delete VM ports attached to this networks before the network
 304             ports = self.neutron.list_ports(network_id=net_id)
 305             for p in ports['ports']:
 306                 try:
 307                     self.neutron.delete_port(p["id"])
 308                 except Exception as e:
 309                     self.logger.error("Error deleting port %s: %s", p["id"], str(e))
 310             self.neutron.delete_network(net_id)
 311             return net_id
 312         except (neExceptions.ConnectionFailed, neExceptions.NetworkNotFoundClient, neExceptions.NeutronException,
 313                 neClient.exceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException) as e:
 314             self._format_exception(e)
 315
 316     def refresh_nets_status(self, net_list):
 317         '''Get the status of the networks
 318            Params: the list of network identifiers
 319            Returns a dictionary with:
 320                 net_id:         #VIM id of this network
 321                     status:     #Mandatory. Text with one of:
 322                                 #  DELETED (not found at vim)
 323                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 324                                 #  OTHER (Vim reported other status not understood)
 325                                 #  ERROR (VIM indicates an ERROR status)
 326                                 #  ACTIVE, INACTIVE, DOWN (admin down),
 327                                 #  BUILD (on building process)
 328                                 #
 329                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 330                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 331
 332         '''
 333         net_dict={}
 334         for net_id in net_list:
 335             net = {}
 336             try:
 337                 net_vim = self.get_network(net_id)
 338                 if net_vim['status'] in netStatus2manoFormat:
 339                     net["status"] = netStatus2manoFormat[ net_vim['status'] ]
 340                 else:
 341                     net["status"] = "OTHER"
 342                     net["error_msg"] = "VIM status reported " + net_vim['status']
 343
 344                 if net['status'] == "ACIVE" and not net_vim['admin_state_up']:
 345                     net['status'] = 'DOWN'
 346                 net['vim_info']  = yaml.safe_dump(net_vim)
 347                 if net_vim.get('fault'):  #TODO
 348                     net['error_msg'] = str(net_vim['fault'])
 349             except vimconn.vimconnNotFoundException as e:
 350                 self.logger.error("Exception getting net status: %s", str(e))
 351                 net['status'] = "DELETED"
 352                 net['error_msg'] = str(e)
 353             except vimconn.vimconnException as e:
 354                 self.logger.error("Exception getting net status: %s", str(e))
 355                 net['status'] = "VIM_ERROR"
 356                 net['error_msg'] = str(e)
 357             net_dict[net_id] = net
 358         return net_dict
 359
 360     def get_flavor(self, flavor_id):
 361         '''Obtain flavor details from the  VIM. Returns the flavor dict details'''
 362         self.logger.debug("Getting flavor '%s'", flavor_id)
 363         try:
 364             self._reload_connection()
 365             flavor = self.nova.flavors.find(id=flavor_id)
 366             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 367             return flavor.to_dict()
 368         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException) as e:
 369             self._format_exception(e)
 370
 371     def new_flavor(self, flavor_data, change_name_if_used=True):
 372         '''Adds a tenant flavor to openstack VIM
 373         if change_name_if_used is True, it will change name in case of conflict, because it is not supported name repetition
 374         Returns the flavor identifier
 375         '''
 376         self.logger.debug("Adding flavor '%s'", str(flavor_data))
 377         retry=0
 378         max_retries=3
 379         name_suffix = 0
 380         name=flavor_data['name']
 381         while retry<max_retries:
 382             retry+=1
 383             try:
 384                 self._reload_connection()
 385                 if change_name_if_used:
 386                     #get used names
 387                     fl_names=[]
 388                     fl=self.nova.flavors.list()
 389                     for f in fl:
 390                         fl_names.append(f.name)
 391                     while name in fl_names:
 392                         name_suffix += 1
 393                         name = flavor_data['name']+"-" + str(name_suffix)
 394
 395                 ram = flavor_data.get('ram',64)
 396                 vcpus = flavor_data.get('vcpus',1)
 397                 numa_properties=None
 398
 399                 extended = flavor_data.get("extended")
 400                 if extended:
 401                     numas=extended.get("numas")
 402                     if numas:
 403                         numa_nodes = len(numas)
 404                         if numa_nodes > 1:
 405                             return -1, "Can not add flavor with more than one numa"
 406                         numa_properties = {"hw:numa_nodes":str(numa_nodes)}
 407                         numa_properties["hw:mem_page_size"] = "large"
 408                         numa_properties["hw:cpu_policy"] = "dedicated"
 409                         numa_properties["hw:numa_mempolicy"] = "strict"
 410                         for numa in numas:
 411                             #overwrite ram and vcpus
 412                             ram = numa['memory']*1024
 413                             if 'paired-threads' in numa:
 414                                 vcpus = numa['paired-threads']*2
 415                                 numa_properties["hw:cpu_threads_policy"] = "prefer"
 416                             elif 'cores' in numa:
 417                                 vcpus = numa['cores']
 418                                 #numa_properties["hw:cpu_threads_policy"] = "prefer"
 419                             elif 'threads' in numa:
 420                                 vcpus = numa['threads']
 421                                 numa_properties["hw:cpu_policy"] = "isolated"
 422                             for interface in numa.get("interfaces",() ):
 423                                 if interface["dedicated"]=="yes":
 424                                     raise vimconn.vimconnException("Passthrough interfaces are not supported for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable)
 425                                 #TODO, add the key 'pci_passthrough:alias"="<label at config>:<number ifaces>"' when a way to connect it is available
 426
 427                 #create flavor
 428                 new_flavor=self.nova.flavors.create(name,
 429                                 ram,
 430                                 vcpus,
 431                                 flavor_data.get('disk',1),
 432                                 is_public=flavor_data.get('is_public', True)
 433                             )
 434                 #add metadata
 435                 if numa_properties:
 436                     new_flavor.set_keys(numa_properties)
 437                 return new_flavor.id
 438             except nvExceptions.Conflict as e:
 439                 if change_name_if_used and retry < max_retries:
 440                     continue
 441                 self._format_exception(e)
 442             #except nvExceptions.BadRequest as e:
 443             except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 444                 self._format_exception(e)
 445
 446     def delete_flavor(self,flavor_id):
 447         '''Deletes a tenant flavor from openstack VIM. Returns the old flavor_id
 448         '''
 449         try:
 450             self._reload_connection()
 451             self.nova.flavors.delete(flavor_id)
 452             return flavor_id
 453         #except nvExceptions.BadRequest as e:
 454         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException) as e:
 455             self._format_exception(e)
 456
 457     def new_image(self,image_dict):
 458         '''
 459         Adds a tenant image to VIM. imge_dict is a dictionary with:
 460             name: name
 461             disk_format: qcow2, vhd, vmdk, raw (by default), ...
 462             location: path or URI
 463             public: "yes" or "no"
 464             metadata: metadata of the image
 465         Returns the image_id
 466         '''
 467         #using version 1 of glance client
 468         glancev1 = gl1Client.Client('1',self.glance_endpoint, token=self.keystone.auth_token, **self.k_creds)  #TODO check k_creds vs n_creds
 469         retry=0
 470         max_retries=3
 471         while retry<max_retries:
 472             retry+=1
 473             try:
 474                 self._reload_connection()
 475                 #determine format  http://docs.openstack.org/developer/glance/formats.html
 476                 if "disk_format" in image_dict:
 477                     disk_format=image_dict["disk_format"]
 478                 else: #autodiscover base on extention
 479                     if image_dict['location'][-6:]==".qcow2":
 480                         disk_format="qcow2"
 481                     elif image_dict['location'][-4:]==".vhd":
 482                         disk_format="vhd"
 483                     elif image_dict['location'][-5:]==".vmdk":
 484                         disk_format="vmdk"
 485                     elif image_dict['location'][-4:]==".vdi":
 486                         disk_format="vdi"
 487                     elif image_dict['location'][-4:]==".iso":
 488                         disk_format="iso"
 489                     elif image_dict['location'][-4:]==".aki":
 490                         disk_format="aki"
 491                     elif image_dict['location'][-4:]==".ari":
 492                         disk_format="ari"
 493                     elif image_dict['location'][-4:]==".ami":
 494                         disk_format="ami"
 495                     else:
 496                         disk_format="raw"
 497                 self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location'])
 498                 if image_dict['location'][0:4]=="http":
 499                     new_image = glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 500                             container_format="bare", location=image_dict['location'], disk_format=disk_format)
 501                 else: #local path
 502                     with open(image_dict['location']) as fimage:
 503                         new_image = glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 504                             container_format="bare", data=fimage, disk_format=disk_format)
 505                 #insert metadata. We cannot use 'new_image.properties.setdefault'
 506                 #because nova and glance are "INDEPENDENT" and we are using nova for reading metadata
 507                 new_image_nova=self.nova.images.find(id=new_image.id)
 508                 new_image_nova.metadata.setdefault('location',image_dict['location'])
 509                 metadata_to_load = image_dict.get('metadata')
 510                 if metadata_to_load:
 511                     for k,v in yaml.load(metadata_to_load).iteritems():
 512                         new_image_nova.metadata.setdefault(k,v)
 513                 return new_image.id
 514             except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e:
 515                 self._format_exception(e)
 516             except (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError) as e:
 517                 if retry==max_retries:
 518                     continue
 519                 self._format_exception(e)
 520             except IOError as e:  #can not open the file
 521                 raise vimconn.vimconnConnectionException(type(e).__name__ + ": " + str(e)+ " for " + image_dict['location'],
 522                                                          http_code=vimconn.HTTP_Bad_Request)
 523
 524     def delete_image(self, image_id):
 525         '''Deletes a tenant image from openstack VIM. Returns the old id
 526         '''
 527         try:
 528             self._reload_connection()
 529             self.nova.images.delete(image_id)
 530             return image_id
 531         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError) as e: #TODO remove
 532             self._format_exception(e)
 533
 534     def get_image_id_from_path(self, path):
 535         '''Get the image id from image path in the VIM database. Returns the image_id
 536         '''
 537         try:
 538             self._reload_connection()
 539             images = self.nova.images.list()
 540             for image in images:
 541                 if image.metadata.get("location")==path:
 542                     return image.id
 543             raise vimconn.vimconnNotFoundException("image with location '{}' not found".format( path))
 544         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError) as e:
 545             self._format_exception(e)
 546
 547     def new_vminstance(self,name,description,start,image_id,flavor_id,net_list,cloud_config=None):
 548         '''Adds a VM instance to VIM
 549         Params:
 550             start: indicates if VM must start or boot in pause mode. Ignored
 551             image_id,flavor_id: iamge and flavor uuid
 552             net_list: list of interfaces, each one is a dictionary with:
 553                 name:
 554                 net_id: network uuid to connect
 555                 vpci: virtual vcpi to assign, ignored because openstack lack #TODO
 556                 model: interface model, ignored #TODO
 557                 mac_address: used for  SR-IOV ifaces #TODO for other types
 558                 use: 'data', 'bridge',  'mgmt'
 559                 type: 'virtual', 'PF', 'VF', 'VFnotShared'
 560                 vim_id: filled/added by this function
 561                 #TODO ip, security groups
 562         Returns the instance identifier
 563         '''
 564         self.logger.debug("Creating VM image '%s' flavor '%s' nics='%s'",image_id, flavor_id,str(net_list))
 565         try:
 566             metadata={}
 567             net_list_vim=[]
 568             self._reload_connection()
 569             metadata_vpci={} #For a specific neutron plugin
 570             for net in net_list:
 571                 if not net.get("net_id"): #skip non connected iface
 572                     continue
 573                 if net["type"]=="virtual":
 574                     net_list_vim.append({'net-id': net["net_id"]})
 575                     if "vpci" in net:
 576                         metadata_vpci[ net["net_id"] ] = [[ net["vpci"], "" ]]
 577                 elif net["type"]=="PF":
 578                     self.logger.warn("new_vminstance: Warning, can not connect a passthrough interface ")
 579                     #TODO insert this when openstack consider passthrough ports as openstack neutron ports
 580                 else: #VF
 581                     if "vpci" in net:
 582                         if "VF" not in metadata_vpci:
 583                             metadata_vpci["VF"]=[]
 584                         metadata_vpci["VF"].append([ net["vpci"], "" ])
 585                     port_dict={
 586                          "network_id": net["net_id"],
 587                          "name": net.get("name"),
 588                          "binding:vnic_type": "direct",
 589                          "admin_state_up": True
 590                     }
 591                     if not port_dict["name"]:
 592                         port_dict["name"] = name
 593                     if net.get("mac_address"):
 594                         port_dict["mac_address"]=net["mac_address"]
 595                     #TODO: manage having SRIOV without vlan tag
 596                     #if net["type"] == "VFnotShared"
 597                     #    port_dict["vlan"]=0
 598                     new_port = self.neutron.create_port({"port": port_dict })
 599                     net["mac_adress"] = new_port["port"]["mac_address"]
 600                     net["vim_id"] = new_port["port"]["id"]
 601                     net["ip"] = new_port["port"].get("fixed_ips",[{}])[0].get("ip_address")
 602                     net_list_vim.append({"port-id": new_port["port"]["id"]})
 603             if metadata_vpci:
 604                 metadata = {"pci_assignement": json.dumps(metadata_vpci)}
 605                 if len(metadata["pci_assignement"]) >255:
 606                     #limit the metadata size
 607                     #metadata["pci_assignement"] = metadata["pci_assignement"][0:255]
 608                     self.logger.warn("Metadata deleted since it exceeds the expected length (255) ")
 609                     metadata = {}
 610
 611             self.logger.debug("name '%s' image_id '%s'flavor_id '%s' net_list_vim '%s' description '%s' metadata %s",
 612                               name, image_id, flavor_id, str(net_list_vim), description, str(metadata))
 613
 614             security_groups   = self.config.get('security_groups')
 615             if type(security_groups) is str:
 616                 security_groups = ( security_groups, )
 617             if isinstance(cloud_config, dict):
 618                 userdata="#cloud-config\nusers:\n"
 619                 #default user
 620                 if "key-pairs" in cloud_config:
 621                     userdata += "  - default:\n    ssh-authorized-keys:\n"
 622                     for key in cloud_config["key-pairs"]:
 623                         userdata += "      - '{key}'\n".format(key=key)
 624                 for user in cloud_config.get("users",[]):
 625                     userdata += "  - name: {name}\n    sudo: ALL=(ALL) NOPASSWD:ALL\n".format(name=user["name"])
 626                     if "user-info" in user:
 627                         userdata += "    gecos: {}'\n".format(user["user-info"])
 628                     if user.get("key-pairs"):
 629                         userdata += "    ssh-authorized-keys:\n"
 630                         for key in user["key-pairs"]:
 631                             userdata += "      - '{key}'\n".format(key=key)
 632                 self.logger.debug("userdata: %s", userdata)
 633             elif isinstance(cloud_config, str):
 634                 userdata = cloud_config
 635             else:
 636                 userdata=None
 637
 638             server = self.nova.servers.create(name, image_id, flavor_id, nics=net_list_vim, meta=metadata,
 639                                               security_groups   = security_groups,
 640                                               availability_zone = self.config.get('availability_zone'),
 641                                               key_name          = self.config.get('keypair'),
 642                                               userdata=userdata
 643                                         ) #, description=description)
 644
 645
 646             #print "DONE :-)", server
 647
 648 #             #TODO   server.add_floating_ip("10.95.87.209")
 649 #             #To look for a free floating_ip
 650 #             free_floating_ip = None
 651 #             for floating_ip in self.neutron.list_floatingips().get("floatingips", () ):
 652 #                 if not floating_ip["port_id"]:
 653 #                     free_floating_ip = floating_ip["floating_ip_address"]
 654 #                     break
 655 #             if free_floating_ip:
 656 #                 server.add_floating_ip(free_floating_ip)
 657
 658
 659             return server.id
 660 #        except nvExceptions.NotFound as e:
 661 #            error_value=-vimconn.HTTP_Not_Found
 662 #            error_text= "vm instance %s not found" % vm_id
 663         except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError,
 664                 neClient.exceptions.ConnectionFailed) as e:
 665             self._format_exception(e)
 666         except TypeError as e:
 667             raise vimconn.vimconnException(type(e).__name__ + ": "+  str(e), http_code=vimconn.HTTP_Bad_Request)
 668
 669     def get_vminstance(self,vm_id):
 670         '''Returns the VM instance information from VIM'''
 671         #self.logger.debug("Getting VM from VIM")
 672         try:
 673             self._reload_connection()
 674             server = self.nova.servers.find(id=vm_id)
 675             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 676             return server.to_dict()
 677         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound) as e:
 678             self._format_exception(e)
 679
 680     def get_vminstance_console(self,vm_id, console_type="vnc"):
 681         '''
 682         Get a console for the virtual machine
 683         Params:
 684             vm_id: uuid of the VM
 685             console_type, can be:
 686                 "novnc" (by default), "xvpvnc" for VNC types,
 687                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
 688         Returns dict with the console parameters:
 689                 protocol: ssh, ftp, http, https, ...
 690                 server:   usually ip address
 691                 port:     the http, ssh, ... port
 692                 suffix:   extra text, e.g. the http path and query string
 693         '''
 694         self.logger.debug("Getting VM CONSOLE from VIM")
 695         try:
 696             self._reload_connection()
 697             server = self.nova.servers.find(id=vm_id)
 698             if console_type == None or console_type == "novnc":
 699                 console_dict = server.get_vnc_console("novnc")
 700             elif console_type == "xvpvnc":
 701                 console_dict = server.get_vnc_console(console_type)
 702             elif console_type == "rdp-html5":
 703                 console_dict = server.get_rdp_console(console_type)
 704             elif console_type == "spice-html5":
 705                 console_dict = server.get_spice_console(console_type)
 706             else:
 707                 raise vimconn.vimconnException("console type '{}' not allowed".format(console_type), http_code=vimconn.HTTP_Bad_Request)
 708
 709             console_dict1 = console_dict.get("console")
 710             if console_dict1:
 711                 console_url = console_dict1.get("url")
 712                 if console_url:
 713                     #parse console_url
 714                     protocol_index = console_url.find("//")
 715                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
 716                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
 717                     if protocol_index < 0 or port_index<0 or suffix_index<0:
 718                         return -vimconn.HTTP_Internal_Server_Error, "Unexpected response from VIM"
 719                     console_dict={"protocol": console_url[0:protocol_index],
 720                                   "server":   console_url[protocol_index+2:port_index],
 721                                   "port":     console_url[port_index:suffix_index],
 722                                   "suffix":   console_url[suffix_index+1:]
 723                                   }
 724                     protocol_index += 2
 725                     return console_dict
 726             raise vimconn.vimconnUnexpectedResponse("Unexpected response from VIM")
 727
 728         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.BadRequest) as e:
 729             self._format_exception(e)
 730
 731     def delete_vminstance(self, vm_id):
 732         '''Removes a VM instance from VIM. Returns the old identifier
 733         '''
 734         #print "osconnector: Getting VM from VIM"
 735         try:
 736             self._reload_connection()
 737             #delete VM ports attached to this networks before the virtual machine
 738             ports = self.neutron.list_ports(device_id=vm_id)
 739             for p in ports['ports']:
 740                 try:
 741                     self.neutron.delete_port(p["id"])
 742                 except Exception as e:
 743                     self.logger.error("Error deleting port: " + type(e).__name__ + ": "+  str(e))
 744             self.nova.servers.delete(vm_id)
 745             return vm_id
 746         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException) as e:
 747             self._format_exception(e)
 748         #TODO insert exception vimconn.HTTP_Unauthorized
 749         #if reaching here is because an exception
 750
 751     def refresh_vms_status(self, vm_list):
 752         '''Get the status of the virtual machines and their interfaces/ports
 753            Params: the list of VM identifiers
 754            Returns a dictionary with:
 755                 vm_id:          #VIM id of this Virtual Machine
 756                     status:     #Mandatory. Text with one of:
 757                                 #  DELETED (not found at vim)
 758                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 759                                 #  OTHER (Vim reported other status not understood)
 760                                 #  ERROR (VIM indicates an ERROR status)
 761                                 #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
 762                                 #  CREATING (on building process), ERROR
 763                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
 764                                 #
 765                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 766                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 767                     interfaces:
 768                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
 769                         mac_address:      #Text format XX:XX:XX:XX:XX:XX
 770                         vim_net_id:       #network id where this interface is connected
 771                         vim_interface_id: #interface/port VIM id
 772                         ip_address:       #null, or text with IPv4, IPv6 address
 773         '''
 774         vm_dict={}
 775         self.logger.debug("refresh_vms status: Getting tenant VM instance information from VIM")
 776         for vm_id in vm_list:
 777             vm={}
 778             try:
 779                 vm_vim = self.get_vminstance(vm_id)
 780                 if vm_vim['status'] in vmStatus2manoFormat:
 781                     vm['status']    =  vmStatus2manoFormat[ vm_vim['status'] ]
 782                 else:
 783                     vm['status']    = "OTHER"
 784                     vm['error_msg'] = "VIM status reported " + vm_vim['status']
 785                 vm['vim_info']  = yaml.safe_dump(vm_vim)
 786                 vm["interfaces"] = []
 787                 if vm_vim.get('fault'):
 788                     vm['error_msg'] = str(vm_vim['fault'])
 789                 #get interfaces
 790                 try:
 791                     self._reload_connection()
 792                     port_dict=self.neutron.list_ports(device_id=vm_id)
 793                     for port in port_dict["ports"]:
 794                         interface={}
 795                         interface['vim_info']  = yaml.safe_dump(port)
 796                         interface["mac_address"] = port.get("mac_address")
 797                         interface["vim_net_id"] = port["network_id"]
 798                         interface["vim_interface_id"] = port["id"]
 799                         ips=[]
 800                         #look for floating ip address
 801                         floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
 802                         if floating_ip_dict.get("floatingips"):
 803                             ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address") )
 804
 805                         for subnet in port["fixed_ips"]:
 806                             ips.append(subnet["ip_address"])
 807                         interface["ip_address"] = ";".join(ips)
 808                         vm["interfaces"].append(interface)
 809                 except Exception as e:
 810                     self.logger.error("Error getting vm interface information " + type(e).__name__ + ": "+  str(e))
 811             except vimconn.vimconnNotFoundException as e:
 812                 self.logger.error("Exception getting vm status: %s", str(e))
 813                 vm['status'] = "DELETED"
 814                 vm['error_msg'] = str(e)
 815             except vimconn.vimconnException as e:
 816                 self.logger.error("Exception getting vm status: %s", str(e))
 817                 vm['status'] = "VIM_ERROR"
 818                 vm['error_msg'] = str(e)
 819             vm_dict[vm_id] = vm
 820         return vm_dict
 821
 822     def action_vminstance(self, vm_id, action_dict):
 823         '''Send and action over a VM instance from VIM
 824         Returns the vm_id if the action was successfully sent to the VIM'''
 825         self.logger.debug("Action over VM '%s': %s", vm_id, str(action_dict))
 826         try:
 827             self._reload_connection()
 828             server = self.nova.servers.find(id=vm_id)
 829             if "start" in action_dict:
 830                 if action_dict["start"]=="rebuild":
 831                     server.rebuild()
 832                 else:
 833                     if server.status=="PAUSED":
 834                         server.unpause()
 835                     elif server.status=="SUSPENDED":
 836                         server.resume()
 837                     elif server.status=="SHUTOFF":
 838                         server.start()
 839             elif "pause" in action_dict:
 840                 server.pause()
 841             elif "resume" in action_dict:
 842                 server.resume()
 843             elif "shutoff" in action_dict or "shutdown" in action_dict:
 844                 server.stop()
 845             elif "forceOff" in action_dict:
 846                 server.stop() #TODO
 847             elif "terminate" in action_dict:
 848                 server.delete()
 849             elif "createImage" in action_dict:
 850                 server.create_image()
 851                 #"path":path_schema,
 852                 #"description":description_schema,
 853                 #"name":name_schema,
 854                 #"metadata":metadata_schema,
 855                 #"imageRef": id_schema,
 856                 #"disk": {"oneOf":[{"type": "null"}, {"type":"string"}] },
 857             elif "rebuild" in action_dict:
 858                 server.rebuild(server.image['id'])
 859             elif "reboot" in action_dict:
 860                 server.reboot() #reboot_type='SOFT'
 861             elif "console" in action_dict:
 862                 console_type = action_dict["console"]
 863                 if console_type == None or console_type == "novnc":
 864                     console_dict = server.get_vnc_console("novnc")
 865                 elif console_type == "xvpvnc":
 866                     console_dict = server.get_vnc_console(console_type)
 867                 elif console_type == "rdp-html5":
 868                     console_dict = server.get_rdp_console(console_type)
 869                 elif console_type == "spice-html5":
 870                     console_dict = server.get_spice_console(console_type)
 871                 else:
 872                     raise vimconn.vimconnException("console type '{}' not allowed".format(console_type),
 873                                                    http_code=vimconn.HTTP_Bad_Request)
 874                 try:
 875                     console_url = console_dict["console"]["url"]
 876                     #parse console_url
 877                     protocol_index = console_url.find("//")
 878                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
 879                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
 880                     if protocol_index < 0 or port_index<0 or suffix_index<0:
 881                         raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
 882                     console_dict2={"protocol": console_url[0:protocol_index],
 883                                   "server":   console_url[protocol_index+2 : port_index],
 884                                   "port":     int(console_url[port_index+1 : suffix_index]),
 885                                   "suffix":   console_url[suffix_index+1:]
 886                                   }
 887                     return console_dict2
 888                 except Exception as e:
 889                     raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
 890
 891             return vm_id
 892         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound) as e:
 893             self._format_exception(e)
 894         #TODO insert exception vimconn.HTTP_Unauthorized
 895
 896 #NOT USED FUNCTIONS
 897
 898     def new_external_port(self, port_data):
 899         #TODO openstack if needed
 900         '''Adds a external port to VIM'''
 901         '''Returns the port identifier'''
 902         return -vimconn.HTTP_Internal_Server_Error, "osconnector.new_external_port() not implemented"
 903
 904     def connect_port_network(self, port_id, network_id, admin=False):
 905         #TODO openstack if needed
 906         '''Connects a external port to a network'''
 907         '''Returns status code of the VIM response'''
 908         return -vimconn.HTTP_Internal_Server_Error, "osconnector.connect_port_network() not implemented"
 909
 910     def new_user(self, user_name, user_passwd, tenant_id=None):
 911         '''Adds a new user to openstack VIM'''
 912         '''Returns the user identifier'''
 913         self.logger.debug("osconnector: Adding a new user to VIM")
 914         try:
 915             self._reload_connection()
 916             user=self.keystone.users.create(user_name, user_passwd, tenant_id=tenant_id)
 917             #self.keystone.tenants.add_user(self.k_creds["username"], #role)
 918             return user.id
 919         except ksExceptions.ConnectionError as e:
 920             error_value=-vimconn.HTTP_Bad_Request
 921             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
 922         except ksExceptions.ClientException as e: #TODO remove
 923             error_value=-vimconn.HTTP_Bad_Request
 924             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
 925         #TODO insert exception vimconn.HTTP_Unauthorized
 926         #if reaching here is because an exception
 927         if self.debug:
 928             self.logger.debug("new_user " + error_text)
 929         return error_value, error_text
 930
 931     def delete_user(self, user_id):
 932         '''Delete a user from openstack VIM'''
 933         '''Returns the user identifier'''
 934         if self.debug:
 935             print "osconnector: Deleting  a  user from VIM"
 936         try:
 937             self._reload_connection()
 938             self.keystone.users.delete(user_id)
 939             return 1, user_id
 940         except ksExceptions.ConnectionError as e:
 941             error_value=-vimconn.HTTP_Bad_Request
 942             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
 943         except ksExceptions.NotFound as e:
 944             error_value=-vimconn.HTTP_Not_Found
 945             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
 946         except ksExceptions.ClientException as e: #TODO remove
 947             error_value=-vimconn.HTTP_Bad_Request
 948             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
 949         #TODO insert exception vimconn.HTTP_Unauthorized
 950         #if reaching here is because an exception
 951         if self.debug:
 952             print "delete_tenant " + error_text
 953         return error_value, error_text
 954
 955     def get_hosts_info(self):
 956         '''Get the information of deployed hosts
 957         Returns the hosts content'''
 958         if self.debug:
 959             print "osconnector: Getting Host info from VIM"
 960         try:
 961             h_list=[]
 962             self._reload_connection()
 963             hypervisors = self.nova.hypervisors.list()
 964             for hype in hypervisors:
 965                 h_list.append( hype.to_dict() )
 966             return 1, {"hosts":h_list}
 967         except nvExceptions.NotFound as e:
 968             error_value=-vimconn.HTTP_Not_Found
 969             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
 970         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
 971             error_value=-vimconn.HTTP_Bad_Request
 972             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
 973         #TODO insert exception vimconn.HTTP_Unauthorized
 974         #if reaching here is because an exception
 975         if self.debug:
 976             print "get_hosts_info " + error_text
 977         return error_value, error_text
 978
 979     def get_hosts(self, vim_tenant):
 980         '''Get the hosts and deployed instances
 981         Returns the hosts content'''
 982         r, hype_dict = self.get_hosts_info()
 983         if r<0:
 984             return r, hype_dict
 985         hypervisors = hype_dict["hosts"]
 986         try:
 987             servers = self.nova.servers.list()
 988             for hype in hypervisors:
 989                 for server in servers:
 990                     if server.to_dict()['OS-EXT-SRV-ATTR:hypervisor_hostname']==hype['hypervisor_hostname']:
 991                         if 'vm' in hype:
 992                             hype['vm'].append(server.id)
 993                         else:
 994                             hype['vm'] = [server.id]
 995             return 1, hype_dict
 996         except nvExceptions.NotFound as e:
 997             error_value=-vimconn.HTTP_Not_Found
 998             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
 999         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1000             error_value=-vimconn.HTTP_Bad_Request
1001             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1002         #TODO insert exception vimconn.HTTP_Unauthorized
1003         #if reaching here is because an exception
1004         if self.debug:
1005             print "get_hosts " + error_text
1006         return error_value, error_text
1007
1008