osm_ro/vimconn_vmware.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2016-2017 VMware Inc.
   5 # This file is part of ETSI OSM
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact:  osslegalrouting@vmware.com
  22 ##
  23
  24 """
  25 vimconn_vmware implementation an Abstract class in order to interact with VMware  vCloud Director.
  26 mbayramov@vmware.com
  27 """
  28 from progressbar import Percentage, Bar, ETA, FileTransferSpeed, ProgressBar
  29
  30 import vimconn
  31 import os
  32 import traceback
  33 import itertools
  34 import requests
  35 import ssl
  36 import atexit
  37
  38 from pyVmomi import vim, vmodl
  39 from pyVim.connect import SmartConnect, Disconnect
  40
  41 from xml.etree import ElementTree as XmlElementTree
  42 from lxml import etree as lxmlElementTree
  43
  44 import yaml
  45 from pyvcloud.vcd.client import BasicLoginCredentials,Client,VcdTaskException
  46 from pyvcloud.vcd.vdc import VDC
  47 from pyvcloud.vcd.org import Org
  48 import re
  49 from pyvcloud.vcd.vapp import VApp
  50 from xml.sax.saxutils import escape
  51 import logging
  52 import json
  53 import time
  54 import uuid
  55 import httplib
  56 #For python3
  57 #import http.client
  58 import hashlib
  59 import socket
  60 import struct
  61 import netaddr
  62 import random
  63
  64 # global variable for vcd connector type
  65 STANDALONE = 'standalone'
  66
  67 # key for flavor dicts
  68 FLAVOR_RAM_KEY = 'ram'
  69 FLAVOR_VCPUS_KEY = 'vcpus'
  70 FLAVOR_DISK_KEY = 'disk'
  71 DEFAULT_IP_PROFILE = {'dhcp_count':50,
  72                       'dhcp_enabled':True,
  73                       'ip_version':"IPv4"
  74                       }
  75 # global variable for wait time
  76 INTERVAL_TIME = 5
  77 MAX_WAIT_TIME = 1800
  78
  79 API_VERSION = '5.9'
  80
  81 __author__ = "Mustafa Bayramov, Arpita Kate, Sachin Bhangare, Prakash Kasar"
  82 __date__ = "$09-Mar-2018 11:09:29$"
  83 __version__ = '0.2'
  84
  85 #     -1: "Could not be created",
  86 #     0: "Unresolved",
  87 #     1: "Resolved",
  88 #     2: "Deployed",
  89 #     3: "Suspended",
  90 #     4: "Powered on",
  91 #     5: "Waiting for user input",
  92 #     6: "Unknown state",
  93 #     7: "Unrecognized state",
  94 #     8: "Powered off",
  95 #     9: "Inconsistent state",
  96 #     10: "Children do not all have the same status",
  97 #     11: "Upload initiated, OVF descriptor pending",
  98 #     12: "Upload initiated, copying contents",
  99 #     13: "Upload initiated , disk contents pending",
 100 #     14: "Upload has been quarantined",
 101 #     15: "Upload quarantine period has expired"
 102
 103 # mapping vCD status to MANO
 104 vcdStatusCode2manoFormat = {4: 'ACTIVE',
 105                             7: 'PAUSED',
 106                             3: 'SUSPENDED',
 107                             8: 'INACTIVE',
 108                             12: 'BUILD',
 109                             -1: 'ERROR',
 110                             14: 'DELETED'}
 111
 112 #
 113 netStatus2manoFormat = {'ACTIVE': 'ACTIVE', 'PAUSED': 'PAUSED', 'INACTIVE': 'INACTIVE', 'BUILD': 'BUILD',
 114                         'ERROR': 'ERROR', 'DELETED': 'DELETED'
 115                         }
 116
 117 class vimconnector(vimconn.vimconnector):
 118     # dict used to store flavor in memory
 119     flavorlist = {}
 120
 121     def __init__(self, uuid=None, name=None, tenant_id=None, tenant_name=None,
 122                  url=None, url_admin=None, user=None, passwd=None, log_level=None, config={}, persistent_info={}):
 123         """
 124         Constructor create vmware connector to vCloud director.
 125
 126         By default construct doesn't validate connection state. So client can create object with None arguments.
 127         If client specified username , password and host and VDC name.  Connector initialize other missing attributes.
 128
 129         a) It initialize organization UUID
 130         b) Initialize tenant_id/vdc ID.   (This information derived from tenant name)
 131
 132         Args:
 133             uuid - is organization uuid.
 134             name - is organization name that must be presented in vCloud director.
 135             tenant_id - is VDC uuid it must be presented in vCloud director
 136             tenant_name - is VDC name.
 137             url - is hostname or ip address of vCloud director
 138             url_admin - same as above.
 139             user - is user that administrator for organization. Caller must make sure that
 140                     username has right privileges.
 141
 142             password - is password for a user.
 143
 144             VMware connector also requires PVDC administrative privileges and separate account.
 145             This variables must be passed via config argument dict contains keys
 146
 147             dict['admin_username']
 148             dict['admin_password']
 149             config - Provide NSX and vCenter information
 150
 151             Returns:
 152                 Nothing.
 153         """
 154
 155         vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url,
 156                                       url_admin, user, passwd, log_level, config)
 157
 158         self.logger = logging.getLogger('openmano.vim.vmware')
 159         self.logger.setLevel(10)
 160         self.persistent_info = persistent_info
 161
 162         self.name = name
 163         self.id = uuid
 164         self.url = url
 165         self.url_admin = url_admin
 166         self.tenant_id = tenant_id
 167         self.tenant_name = tenant_name
 168         self.user = user
 169         self.passwd = passwd
 170         self.config = config
 171         self.admin_password = None
 172         self.admin_user = None
 173         self.org_name = ""
 174         self.nsx_manager = None
 175         self.nsx_user = None
 176         self.nsx_password = None
 177
 178         # Disable warnings from self-signed certificates.
 179         requests.packages.urllib3.disable_warnings()
 180
 181         if tenant_name is not None:
 182             orgnameandtenant = tenant_name.split(":")
 183             if len(orgnameandtenant) == 2:
 184                 self.tenant_name = orgnameandtenant[1]
 185                 self.org_name = orgnameandtenant[0]
 186             else:
 187                 self.tenant_name = tenant_name
 188         if "orgname" in config:
 189             self.org_name = config['orgname']
 190
 191         if log_level:
 192             self.logger.setLevel(getattr(logging, log_level))
 193
 194         try:
 195             self.admin_user = config['admin_username']
 196             self.admin_password = config['admin_password']
 197         except KeyError:
 198             raise vimconn.vimconnException(message="Error admin username or admin password is empty.")
 199
 200         try:
 201             self.nsx_manager = config['nsx_manager']
 202             self.nsx_user = config['nsx_user']
 203             self.nsx_password = config['nsx_password']
 204         except KeyError:
 205             raise vimconn.vimconnException(message="Error: nsx manager or nsx user or nsx password is empty in Config")
 206
 207         self.vcenter_ip = config.get("vcenter_ip", None)
 208         self.vcenter_port = config.get("vcenter_port", None)
 209         self.vcenter_user = config.get("vcenter_user", None)
 210         self.vcenter_password = config.get("vcenter_password", None)
 211
 212 # ############# Stub code for SRIOV #################
 213 #         try:
 214 #             self.dvs_name = config['dv_switch_name']
 215 #         except KeyError:
 216 #             raise vimconn.vimconnException(message="Error: distributed virtaul switch name is empty in Config")
 217 #
 218 #         self.vlanID_range = config.get("vlanID_range", None)
 219
 220         self.org_uuid = None
 221         self.client = None
 222
 223         if not url:
 224             raise vimconn.vimconnException('url param can not be NoneType')
 225
 226         if not self.url_admin:  # try to use normal url
 227             self.url_admin = self.url
 228
 229         logging.debug("UUID: {} name: {} tenant_id: {} tenant name {}".format(self.id, self.org_name,
 230                                                                               self.tenant_id, self.tenant_name))
 231         logging.debug("vcd url {} vcd username: {} vcd password: {}".format(self.url, self.user, self.passwd))
 232         logging.debug("vcd admin username {} vcd admin passowrd {}".format(self.admin_user, self.admin_password))
 233
 234         # initialize organization
 235         if self.user is not None and self.passwd is not None and self.url:
 236             self.init_organization()
 237
 238     def __getitem__(self, index):
 239         if index == 'name':
 240             return self.name
 241         if index == 'tenant_id':
 242             return self.tenant_id
 243         if index == 'tenant_name':
 244             return self.tenant_name
 245         elif index == 'id':
 246             return self.id
 247         elif index == 'org_name':
 248             return self.org_name
 249         elif index == 'org_uuid':
 250             return self.org_uuid
 251         elif index == 'user':
 252             return self.user
 253         elif index == 'passwd':
 254             return self.passwd
 255         elif index == 'url':
 256             return self.url
 257         elif index == 'url_admin':
 258             return self.url_admin
 259         elif index == "config":
 260             return self.config
 261         else:
 262             raise KeyError("Invalid key '%s'" % str(index))
 263
 264     def __setitem__(self, index, value):
 265         if index == 'name':
 266             self.name = value
 267         if index == 'tenant_id':
 268             self.tenant_id = value
 269         if index == 'tenant_name':
 270             self.tenant_name = value
 271         elif index == 'id':
 272             self.id = value
 273         elif index == 'org_name':
 274             self.org_name = value
 275         elif index == 'org_uuid':
 276             self.org_uuid = value
 277         elif index == 'user':
 278             self.user = value
 279         elif index == 'passwd':
 280             self.passwd = value
 281         elif index == 'url':
 282             self.url = value
 283         elif index == 'url_admin':
 284             self.url_admin = value
 285         else:
 286             raise KeyError("Invalid key '%s'" % str(index))
 287
 288     def connect_as_admin(self):
 289         """ Method connect as pvdc admin user to vCloud director.
 290             There are certain action that can be done only by provider vdc admin user.
 291             Organization creation / provider network creation etc.
 292
 293             Returns:
 294                 The return client object that latter can be used to connect to vcloud director as admin for provider vdc
 295         """
 296
 297         self.logger.debug("Logging into vCD {} as admin.".format(self.org_name))
 298
 299         try:
 300             host = self.url
 301             org = 'System'
 302             client_as_admin = Client(host, verify_ssl_certs=False)
 303             client_as_admin.set_credentials(BasicLoginCredentials(self.admin_user, org, self.admin_password))
 304         except Exception as e:
 305             raise vimconn.vimconnException(
 306                   "Can't connect to a vCloud director as: {} with exception {}".format(self.admin_user, e))
 307
 308         return client_as_admin
 309
 310     def connect(self):
 311         """ Method connect as normal user to vCloud director.
 312
 313             Returns:
 314                 The return client object that latter can be used to connect to vCloud director as admin for VDC
 315         """
 316
 317         try:
 318             self.logger.debug("Logging into vCD {} as {} to datacenter {}.".format(self.org_name,
 319                                                                                       self.user,
 320                                                                                       self.org_name))
 321             host = self.url
 322             client = Client(host, verify_ssl_certs=False)
 323             client.set_credentials(BasicLoginCredentials(self.user, self.org_name, self.passwd))
 324         except:
 325             raise vimconn.vimconnConnectionException("Can't connect to a vCloud director org: "
 326                                                      "{} as user: {}".format(self.org_name, self.user))
 327
 328         return client
 329
 330     def init_organization(self):
 331         """ Method initialize organization UUID and VDC parameters.
 332
 333             At bare minimum client must provide organization name that present in vCloud director and VDC.
 334
 335             The VDC - UUID ( tenant_id) will be initialized at the run time if client didn't call constructor.
 336             The Org - UUID will be initialized at the run time if data center present in vCloud director.
 337
 338             Returns:
 339                 The return vca object that letter can be used to connect to vcloud direct as admin
 340         """
 341         client = self.connect()
 342         if not client:
 343             raise vimconn.vimconnConnectionException("Failed to connect vCD.")
 344
 345         self.client = client
 346         try:
 347             if self.org_uuid is None:
 348                 org_list = client.get_org_list()
 349                 for org in org_list.Org:
 350                     # we set org UUID at the init phase but we can do it only when we have valid credential.
 351                     if org.get('name') == self.org_name:
 352                         self.org_uuid = org.get('href').split('/')[-1]
 353                         self.logger.debug("Setting organization UUID {}".format(self.org_uuid))
 354                         break
 355                 else:
 356                     raise vimconn.vimconnException("Vcloud director organization {} not found".format(self.org_name))
 357
 358                 # if well good we require for org details
 359                 org_details_dict = self.get_org(org_uuid=self.org_uuid)
 360
 361                 # we have two case if we want to initialize VDC ID or VDC name at run time
 362                 # tenant_name provided but no tenant id
 363                 if self.tenant_id is None and self.tenant_name is not None and 'vdcs' in org_details_dict:
 364                     vdcs_dict = org_details_dict['vdcs']
 365                     for vdc in vdcs_dict:
 366                         if vdcs_dict[vdc] == self.tenant_name:
 367                             self.tenant_id = vdc
 368                             self.logger.debug("Setting vdc uuid {} for organization UUID {}".format(self.tenant_id,
 369                                                                                                     self.org_name))
 370                             break
 371                     else:
 372                         raise vimconn.vimconnException("Tenant name indicated but not present in vcloud director.")
 373                     # case two we have tenant_id but we don't have tenant name so we find and set it.
 374                     if self.tenant_id is not None and self.tenant_name is None and 'vdcs' in org_details_dict:
 375                         vdcs_dict = org_details_dict['vdcs']
 376                         for vdc in vdcs_dict:
 377                             if vdc == self.tenant_id:
 378                                 self.tenant_name = vdcs_dict[vdc]
 379                                 self.logger.debug("Setting vdc uuid {} for organization UUID {}".format(self.tenant_id,
 380                                                                                                         self.org_name))
 381                                 break
 382                         else:
 383                             raise vimconn.vimconnException("Tenant id indicated but not present in vcloud director")
 384             self.logger.debug("Setting organization uuid {}".format(self.org_uuid))
 385         except:
 386             self.logger.debug("Failed initialize organization UUID for org {}".format(self.org_name))
 387             self.logger.debug(traceback.format_exc())
 388             self.org_uuid = None
 389
 390     def new_tenant(self, tenant_name=None, tenant_description=None):
 391         """ Method adds a new tenant to VIM with this name.
 392             This action requires access to create VDC action in vCloud director.
 393
 394             Args:
 395                 tenant_name is tenant_name to be created.
 396                 tenant_description not used for this call
 397
 398             Return:
 399                 returns the tenant identifier in UUID format.
 400                 If action is failed method will throw vimconn.vimconnException method
 401             """
 402         vdc_task = self.create_vdc(vdc_name=tenant_name)
 403         if vdc_task is not None:
 404             vdc_uuid, value = vdc_task.popitem()
 405             self.logger.info("Created new vdc {} and uuid: {}".format(tenant_name, vdc_uuid))
 406             return vdc_uuid
 407         else:
 408             raise vimconn.vimconnException("Failed create tenant {}".format(tenant_name))
 409
 410     def delete_tenant(self, tenant_id=None):
 411         """ Delete a tenant from VIM
 412              Args:
 413                 tenant_id is tenant_id to be deleted.
 414
 415             Return:
 416                 returns the tenant identifier in UUID format.
 417                 If action is failed method will throw exception
 418         """
 419         vca = self.connect_as_admin()
 420         if not vca:
 421             raise vimconn.vimconnConnectionException("Failed to connect vCD")
 422
 423         if tenant_id is not None:
 424             if vca._session:
 425                 #Get OrgVDC
 426                 url_list = [self.url, '/api/vdc/', tenant_id]
 427                 orgvdc_herf = ''.join(url_list)
 428
 429                 headers = {'Accept':'application/*+xml;version=' + API_VERSION,
 430                            'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
 431                 response = self.perform_request(req_type='GET',
 432                                                 url=orgvdc_herf,
 433                                                 headers=headers)
 434
 435                 if response.status_code != requests.codes.ok:
 436                     self.logger.debug("delete_tenant():GET REST API call {} failed. "\
 437                                       "Return status code {}".format(orgvdc_herf,
 438                                                                      response.status_code))
 439                     raise vimconn.vimconnNotFoundException("Fail to get tenant {}".format(tenant_id))
 440
 441                 lxmlroot_respond = lxmlElementTree.fromstring(response.content)
 442                 namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.iteritems() if prefix}
 443                 #For python3
 444                 #namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.items() if prefix}
 445                 namespaces["xmlns"]= "http://www.vmware.com/vcloud/v1.5"
 446                 vdc_remove_href = lxmlroot_respond.find("xmlns:Link[@rel='remove']",namespaces).attrib['href']
 447                 vdc_remove_href = vdc_remove_href + '?recursive=true&force=true'
 448
 449                 response = self.perform_request(req_type='DELETE',
 450                                                 url=vdc_remove_href,
 451                                                 headers=headers)
 452
 453                 if response.status_code == 202:
 454                     time.sleep(5)
 455                     return tenant_id
 456                 else:
 457                     self.logger.debug("delete_tenant(): DELETE REST API call {} failed. "\
 458                                       "Return status code {}".format(vdc_remove_href,
 459                                                                      response.status_code))
 460                     raise vimconn.vimconnException("Fail to delete tenant with ID {}".format(tenant_id))
 461         else:
 462             self.logger.debug("delete_tenant():Incorrect tenant ID  {}".format(tenant_id))
 463             raise vimconn.vimconnNotFoundException("Fail to get tenant {}".format(tenant_id))
 464
 465
 466     def get_tenant_list(self, filter_dict={}):
 467         """Obtain tenants of VIM
 468         filter_dict can contain the following keys:
 469             name: filter by tenant name
 470             id: filter by tenant uuid/id
 471             <other VIM specific>
 472         Returns the tenant list of dictionaries:
 473             [{'name':'<name>, 'id':'<id>, ...}, ...]
 474
 475         """
 476         org_dict = self.get_org(self.org_uuid)
 477         vdcs_dict = org_dict['vdcs']
 478
 479         vdclist = []
 480         try:
 481             for k in vdcs_dict:
 482                 entry = {'name': vdcs_dict[k], 'id': k}
 483                 # if caller didn't specify dictionary we return all tenants.
 484                 if filter_dict is not None and filter_dict:
 485                     filtered_entry = entry.copy()
 486                     filtered_dict = set(entry.keys()) - set(filter_dict)
 487                     for unwanted_key in filtered_dict: del entry[unwanted_key]
 488                     if filter_dict == entry:
 489                         vdclist.append(filtered_entry)
 490                 else:
 491                     vdclist.append(entry)
 492         except:
 493             self.logger.debug("Error in get_tenant_list()")
 494             self.logger.debug(traceback.format_exc())
 495             raise vimconn.vimconnException("Incorrect state. {}")
 496
 497         return vdclist
 498
 499     def new_network(self, net_name, net_type, ip_profile=None, shared=False):
 500         """Adds a tenant network to VIM
 501             net_name is the name
 502             net_type can be 'bridge','data'.'ptp'.
 503             ip_profile is a dict containing the IP parameters of the network
 504             shared is a boolean
 505         Returns the network identifier"""
 506
 507         self.logger.debug("new_network tenant {} net_type {} ip_profile {} shared {}"
 508                           .format(net_name, net_type, ip_profile, shared))
 509
 510         isshared = 'false'
 511         if shared:
 512             isshared = 'true'
 513
 514 # ############# Stub code for SRIOV #################
 515 #         if net_type == "data" or net_type == "ptp":
 516 #             if self.config.get('dv_switch_name') == None:
 517 #                  raise vimconn.vimconnConflictException("You must provide 'dv_switch_name' at config value")
 518 #             network_uuid = self.create_dvPort_group(net_name)
 519
 520         network_uuid = self.create_network(network_name=net_name, net_type=net_type,
 521                                            ip_profile=ip_profile, isshared=isshared)
 522         if network_uuid is not None:
 523             return network_uuid
 524         else:
 525             raise vimconn.vimconnUnexpectedResponse("Failed create a new network {}".format(net_name))
 526
 527     def get_vcd_network_list(self):
 528         """ Method available organization for a logged in tenant
 529
 530             Returns:
 531                 The return vca object that letter can be used to connect to vcloud direct as admin
 532         """
 533
 534         self.logger.debug("get_vcd_network_list(): retrieving network list for vcd {}".format(self.tenant_name))
 535
 536         if not self.tenant_name:
 537             raise vimconn.vimconnConnectionException("Tenant name is empty.")
 538
 539         org, vdc = self.get_vdc_details()
 540         if vdc is None:
 541             raise vimconn.vimconnConnectionException("Can't retrieve information for a VDC {}".format(self.tenant_name))
 542
 543         vdc_uuid = vdc.get('id').split(":")[3]
 544         if self.client._session:
 545                 headers = {'Accept':'application/*+xml;version=' + API_VERSION,
 546                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
 547                 response = self.perform_request(req_type='GET',
 548                                            url=vdc.get('href'),
 549                                                headers=headers)
 550         if response.status_code != 200:
 551             self.logger.error("Failed to get vdc content")
 552             raise vimconn.vimconnNotFoundException("Failed to get vdc content")
 553         else:
 554             content = XmlElementTree.fromstring(response.content)
 555
 556         network_list = []
 557         try:
 558             for item in content:
 559                 if item.tag.split('}')[-1] == 'AvailableNetworks':
 560                     for net in item:
 561                         response = self.perform_request(req_type='GET',
 562                                                    url=net.get('href'),
 563                                                        headers=headers)
 564
 565                         if response.status_code != 200:
 566                             self.logger.error("Failed to get network content")
 567                             raise vimconn.vimconnNotFoundException("Failed to get network content")
 568                         else:
 569                             net_details = XmlElementTree.fromstring(response.content)
 570
 571                             filter_dict = {}
 572                             net_uuid = net_details.get('id').split(":")
 573                             if len(net_uuid) != 4:
 574                                 continue
 575                             else:
 576                                 net_uuid = net_uuid[3]
 577                                 # create dict entry
 578                                 self.logger.debug("Adding  {} to a list vcd id {} network {}".format(net_uuid,
 579                                                                                                         vdc_uuid,
 580                                                                                          net_details.get('name')))
 581                                 filter_dict["name"] = net_details.get('name')
 582                                 filter_dict["id"] = net_uuid
 583                                 if [i.text for i in net_details if i.tag.split('}')[-1] == 'IsShared'][0] == 'true':
 584                                     shared = True
 585                                 else:
 586                                     shared = False
 587                                 filter_dict["shared"] = shared
 588                                 filter_dict["tenant_id"] = vdc_uuid
 589                                 if net_details.get('status') == 1:
 590                                     filter_dict["admin_state_up"] = True
 591                                 else:
 592                                     filter_dict["admin_state_up"] = False
 593                                 filter_dict["status"] = "ACTIVE"
 594                                 filter_dict["type"] = "bridge"
 595                                 network_list.append(filter_dict)
 596                                 self.logger.debug("get_vcd_network_list adding entry {}".format(filter_dict))
 597         except:
 598             self.logger.debug("Error in get_vcd_network_list", exc_info=True)
 599             pass
 600
 601         self.logger.debug("get_vcd_network_list returning {}".format(network_list))
 602         return network_list
 603
 604     def get_network_list(self, filter_dict={}):
 605         """Obtain tenant networks of VIM
 606         Filter_dict can be:
 607             name: network name  OR/AND
 608             id: network uuid    OR/AND
 609             shared: boolean     OR/AND
 610             tenant_id: tenant   OR/AND
 611             admin_state_up: boolean
 612             status: 'ACTIVE'
 613
 614         [{key : value , key : value}]
 615
 616         Returns the network list of dictionaries:
 617             [{<the fields at Filter_dict plus some VIM specific>}, ...]
 618             List can be empty
 619         """
 620
 621         self.logger.debug("get_network_list(): retrieving network list for vcd {}".format(self.tenant_name))
 622
 623         if not self.tenant_name:
 624             raise vimconn.vimconnConnectionException("Tenant name is empty.")
 625
 626         org, vdc = self.get_vdc_details()
 627         if vdc is None:
 628             raise vimconn.vimconnConnectionException("Can't retrieve information for a VDC {}.".format(self.tenant_name))
 629
 630         try:
 631             vdcid = vdc.get('id').split(":")[3]
 632
 633             if self.client._session:
 634                 headers = {'Accept':'application/*+xml;version=' + API_VERSION,
 635                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
 636                 response = self.perform_request(req_type='GET',
 637                                            url=vdc.get('href'),
 638                                                headers=headers)
 639             if response.status_code != 200:
 640                 self.logger.error("Failed to get vdc content")
 641                 raise vimconn.vimconnNotFoundException("Failed to get vdc content")
 642             else:
 643                 content = XmlElementTree.fromstring(response.content)
 644
 645             network_list = []
 646             for item in content:
 647                 if item.tag.split('}')[-1] == 'AvailableNetworks':
 648                     for net in item:
 649                         response = self.perform_request(req_type='GET',
 650                                                    url=net.get('href'),
 651                                                        headers=headers)
 652
 653                         if response.status_code != 200:
 654                             self.logger.error("Failed to get network content")
 655                             raise vimconn.vimconnNotFoundException("Failed to get network content")
 656                         else:
 657                             net_details = XmlElementTree.fromstring(response.content)
 658
 659                             filter_entry = {}
 660                             net_uuid = net_details.get('id').split(":")
 661                             if len(net_uuid) != 4:
 662                                 continue
 663                             else:
 664                                 net_uuid = net_uuid[3]
 665                                 # create dict entry
 666                                 self.logger.debug("Adding  {} to a list vcd id {} network {}".format(net_uuid,
 667                                                                                                         vdcid,
 668                                                                                          net_details.get('name')))
 669                                 filter_entry["name"] = net_details.get('name')
 670                                 filter_entry["id"] = net_uuid
 671                                 if [i.text for i in net_details if i.tag.split('}')[-1] == 'IsShared'][0] == 'true':
 672                                     shared = True
 673                                 else:
 674                                     shared = False
 675                                 filter_entry["shared"] = shared
 676                                 filter_entry["tenant_id"] = vdcid
 677                                 if net_details.get('status') == 1:
 678                                     filter_entry["admin_state_up"] = True
 679                                 else:
 680                                     filter_entry["admin_state_up"] = False
 681                                 filter_entry["status"] = "ACTIVE"
 682                                 filter_entry["type"] = "bridge"
 683                                 filtered_entry = filter_entry.copy()
 684
 685                                 if filter_dict is not None and filter_dict:
 686                                     # we remove all the key : value we don't care and match only
 687                                     # respected field
 688                                     filtered_dict = set(filter_entry.keys()) - set(filter_dict)
 689                                     for unwanted_key in filtered_dict: del filter_entry[unwanted_key]
 690                                     if filter_dict == filter_entry:
 691                                         network_list.append(filtered_entry)
 692                                 else:
 693                                     network_list.append(filtered_entry)
 694         except Exception as e:
 695             self.logger.debug("Error in get_network_list",exc_info=True)
 696             if isinstance(e, vimconn.vimconnException):
 697                 raise
 698             else:
 699                 raise vimconn.vimconnNotFoundException("Failed : Networks list not found {} ".format(e))
 700
 701         self.logger.debug("Returning {}".format(network_list))
 702         return network_list
 703
 704     def get_network(self, net_id):
 705         """Method obtains network details of net_id VIM network
 706            Return a dict with  the fields at filter_dict (see get_network_list) plus some VIM specific>}, ...]"""
 707
 708         try:
 709             org, vdc = self.get_vdc_details()
 710             vdc_id = vdc.get('id').split(":")[3]
 711             if self.client._session:
 712                 headers = {'Accept':'application/*+xml;version=' + API_VERSION,
 713                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
 714                 response = self.perform_request(req_type='GET',
 715                                            url=vdc.get('href'),
 716                                                headers=headers)
 717             if response.status_code != 200:
 718                 self.logger.error("Failed to get vdc content")
 719                 raise vimconn.vimconnNotFoundException("Failed to get vdc content")
 720             else:
 721                 content = XmlElementTree.fromstring(response.content)
 722
 723             filter_dict = {}
 724
 725             for item in content:
 726                 if item.tag.split('}')[-1] == 'AvailableNetworks':
 727                     for net in item:
 728                         response = self.perform_request(req_type='GET',
 729                                                    url=net.get('href'),
 730                                                        headers=headers)
 731
 732                         if response.status_code != 200:
 733                             self.logger.error("Failed to get network content")
 734                             raise vimconn.vimconnNotFoundException("Failed to get network content")
 735                         else:
 736                             net_details = XmlElementTree.fromstring(response.content)
 737
 738                             vdc_network_id = net_details.get('id').split(":")
 739                             if len(vdc_network_id) == 4 and vdc_network_id[3] == net_id:
 740                                 filter_dict["name"] = net_details.get('name')
 741                                 filter_dict["id"] = vdc_network_id[3]
 742                                 if [i.text for i in net_details if i.tag.split('}')[-1] == 'IsShared'][0] == 'true':
 743                                     shared = True
 744                                 else:
 745                                     shared = False
 746                                 filter_dict["shared"] = shared
 747                                 filter_dict["tenant_id"] = vdc_id
 748                                 if net_details.get('status') == 1:
 749                                     filter_dict["admin_state_up"] = True
 750                                 else:
 751                                     filter_dict["admin_state_up"] = False
 752                                 filter_dict["status"] = "ACTIVE"
 753                                 filter_dict["type"] = "bridge"
 754                                 self.logger.debug("Returning {}".format(filter_dict))
 755                                 return filter_dict
 756                     else:
 757                         raise vimconn.vimconnNotFoundException("Network {} not found".format(net_id))
 758         except Exception as e:
 759             self.logger.debug("Error in get_network")
 760             self.logger.debug(traceback.format_exc())
 761             if isinstance(e, vimconn.vimconnException):
 762                 raise
 763             else:
 764                 raise vimconn.vimconnNotFoundException("Failed : Network not found {} ".format(e))
 765
 766         return filter_dict
 767
 768     def delete_network(self, net_id):
 769         """
 770             Method Deletes a tenant network from VIM, provide the network id.
 771
 772             Returns the network identifier or raise an exception
 773         """
 774
 775         # ############# Stub code for SRIOV #################
 776 #         dvport_group = self.get_dvport_group(net_id)
 777 #         if dvport_group:
 778 #             #delete portgroup
 779 #             status = self.destroy_dvport_group(net_id)
 780 #             if status:
 781 #                 # Remove vlanID from persistent info
 782 #                 if net_id in self.persistent_info["used_vlanIDs"]:
 783 #                     del self.persistent_info["used_vlanIDs"][net_id]
 784 #
 785 #                 return net_id
 786
 787         vcd_network = self.get_vcd_network(network_uuid=net_id)
 788         if vcd_network is not None and vcd_network:
 789             if self.delete_network_action(network_uuid=net_id):
 790                 return net_id
 791         else:
 792             raise vimconn.vimconnNotFoundException("Network {} not found".format(net_id))
 793
 794     def refresh_nets_status(self, net_list):
 795         """Get the status of the networks
 796            Params: the list of network identifiers
 797            Returns a dictionary with:
 798                 net_id:         #VIM id of this network
 799                     status:     #Mandatory. Text with one of:
 800                                 #  DELETED (not found at vim)
 801                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 802                                 #  OTHER (Vim reported other status not understood)
 803                                 #  ERROR (VIM indicates an ERROR status)
 804                                 #  ACTIVE, INACTIVE, DOWN (admin down),
 805                                 #  BUILD (on building process)
 806                                 #
 807                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 808                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 809
 810         """
 811
 812         dict_entry = {}
 813         try:
 814             for net in net_list:
 815                 errormsg = ''
 816                 vcd_network = self.get_vcd_network(network_uuid=net)
 817                 if vcd_network is not None and vcd_network:
 818                     if vcd_network['status'] == '1':
 819                         status = 'ACTIVE'
 820                     else:
 821                         status = 'DOWN'
 822                 else:
 823                     status = 'DELETED'
 824                     errormsg = 'Network not found.'
 825
 826                 dict_entry[net] = {'status': status, 'error_msg': errormsg,
 827                                    'vim_info': yaml.safe_dump(vcd_network)}
 828         except:
 829             self.logger.debug("Error in refresh_nets_status")
 830             self.logger.debug(traceback.format_exc())
 831
 832         return dict_entry
 833
 834     def get_flavor(self, flavor_id):
 835         """Obtain flavor details from the  VIM
 836             Returns the flavor dict details {'id':<>, 'name':<>, other vim specific } #TODO to concrete
 837         """
 838         if flavor_id not in vimconnector.flavorlist:
 839             raise vimconn.vimconnNotFoundException("Flavor not found.")
 840         return vimconnector.flavorlist[flavor_id]
 841
 842     def new_flavor(self, flavor_data):
 843         """Adds a tenant flavor to VIM
 844             flavor_data contains a dictionary with information, keys:
 845                 name: flavor name
 846                 ram: memory (cloud type) in MBytes
 847                 vpcus: cpus (cloud type)
 848                 extended: EPA parameters
 849                   - numas: #items requested in same NUMA
 850                         memory: number of 1G huge pages memory
 851                         paired-threads|cores|threads: number of paired hyperthreads, complete cores OR individual threads
 852                         interfaces: # passthrough(PT) or SRIOV interfaces attached to this numa
 853                           - name: interface name
 854                             dedicated: yes|no|yes:sriov;  for PT, SRIOV or only one SRIOV for the physical NIC
 855                             bandwidth: X Gbps; requested guarantee bandwidth
 856                             vpci: requested virtual PCI address
 857                 disk: disk size
 858                 is_public:
 859                  #TODO to concrete
 860         Returns the flavor identifier"""
 861
 862         # generate a new uuid put to internal dict and return it.
 863         self.logger.debug("Creating new flavor - flavor_data: {}".format(flavor_data))
 864         new_flavor=flavor_data
 865         ram = flavor_data.get(FLAVOR_RAM_KEY, 1024)
 866         cpu = flavor_data.get(FLAVOR_VCPUS_KEY, 1)
 867         disk = flavor_data.get(FLAVOR_DISK_KEY, 0)
 868
 869         if not isinstance(ram, int):
 870             raise vimconn.vimconnException("Non-integer value for ram")
 871         elif not isinstance(cpu, int):
 872             raise vimconn.vimconnException("Non-integer value for cpu")
 873         elif not isinstance(disk, int):
 874             raise vimconn.vimconnException("Non-integer value for disk")
 875
 876         extended_flv = flavor_data.get("extended")
 877         if extended_flv:
 878             numas=extended_flv.get("numas")
 879             if numas:
 880                 for numa in numas:
 881                     #overwrite ram and vcpus
 882                     ram = numa['memory']*1024
 883                     if 'paired-threads' in numa:
 884                         cpu = numa['paired-threads']*2
 885                     elif 'cores' in numa:
 886                         cpu = numa['cores']
 887                     elif 'threads' in numa:
 888                         cpu = numa['threads']
 889
 890         new_flavor[FLAVOR_RAM_KEY] = ram
 891         new_flavor[FLAVOR_VCPUS_KEY] = cpu
 892         new_flavor[FLAVOR_DISK_KEY] = disk
 893         # generate a new uuid put to internal dict and return it.
 894         flavor_id = uuid.uuid4()
 895         vimconnector.flavorlist[str(flavor_id)] = new_flavor
 896         self.logger.debug("Created flavor - {} : {}".format(flavor_id, new_flavor))
 897
 898         return str(flavor_id)
 899
 900     def delete_flavor(self, flavor_id):
 901         """Deletes a tenant flavor from VIM identify by its id
 902
 903            Returns the used id or raise an exception
 904         """
 905         if flavor_id not in vimconnector.flavorlist:
 906             raise vimconn.vimconnNotFoundException("Flavor not found.")
 907
 908         vimconnector.flavorlist.pop(flavor_id, None)
 909         return flavor_id
 910
 911     def new_image(self, image_dict):
 912         """
 913         Adds a tenant image to VIM
 914         Returns:
 915             200, image-id        if the image is created
 916             <0, message          if there is an error
 917         """
 918
 919         return self.get_image_id_from_path(image_dict['location'])
 920
 921     def delete_image(self, image_id):
 922         """
 923             Deletes a tenant image from VIM
 924             Args:
 925                 image_id is ID of Image to be deleted
 926             Return:
 927                 returns the image identifier in UUID format or raises an exception on error
 928         """
 929         conn = self.connect_as_admin()
 930         if not conn:
 931             raise vimconn.vimconnConnectionException("Failed to connect vCD")
 932         # Get Catalog details
 933         url_list = [self.url, '/api/catalog/', image_id]
 934         catalog_herf = ''.join(url_list)
 935
 936         headers = {'Accept':'application/*+xml;version=' + API_VERSION,
 937                   'x-vcloud-authorization': conn._session.headers['x-vcloud-authorization']}
 938
 939         response = self.perform_request(req_type='GET',
 940                                         url=catalog_herf,
 941                                         headers=headers)
 942
 943         if response.status_code != requests.codes.ok:
 944             self.logger.debug("delete_image():GET REST API call {} failed. "\
 945                               "Return status code {}".format(catalog_herf,
 946                                                              response.status_code))
 947             raise vimconn.vimconnNotFoundException("Fail to get image {}".format(image_id))
 948
 949         lxmlroot_respond = lxmlElementTree.fromstring(response.content)
 950         namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.iteritems() if prefix}
 951         #For python3
 952         #namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.items() if prefix}
 953         namespaces["xmlns"]= "http://www.vmware.com/vcloud/v1.5"
 954
 955         catalogItems_section = lxmlroot_respond.find("xmlns:CatalogItems",namespaces)
 956         catalogItems = catalogItems_section.iterfind("xmlns:CatalogItem",namespaces)
 957         for catalogItem in catalogItems:
 958             catalogItem_href = catalogItem.attrib['href']
 959
 960             response = self.perform_request(req_type='GET',
 961                                         url=catalogItem_href,
 962                                         headers=headers)
 963
 964             if response.status_code != requests.codes.ok:
 965                 self.logger.debug("delete_image():GET REST API call {} failed. "\
 966                                   "Return status code {}".format(catalog_herf,
 967                                                                  response.status_code))
 968                 raise vimconn.vimconnNotFoundException("Fail to get catalogItem {} for catalog {}".format(
 969                                                                                     catalogItem,
 970                                                                                     image_id))
 971
 972             lxmlroot_respond = lxmlElementTree.fromstring(response.content)
 973             namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.iteritems() if prefix}
 974             #For python3
 975             #namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.items() if prefix}
 976             namespaces["xmlns"]= "http://www.vmware.com/vcloud/v1.5"
 977             catalogitem_remove_href = lxmlroot_respond.find("xmlns:Link[@rel='remove']",namespaces).attrib['href']
 978
 979             #Remove catalogItem
 980             response = self.perform_request(req_type='DELETE',
 981                                         url=catalogitem_remove_href,
 982                                         headers=headers)
 983             if response.status_code == requests.codes.no_content:
 984                 self.logger.debug("Deleted Catalog item {}".format(catalogItem))
 985             else:
 986                 raise vimconn.vimconnException("Fail to delete Catalog Item {}".format(catalogItem))
 987
 988         #Remove catalog
 989         url_list = [self.url, '/api/admin/catalog/', image_id]
 990         catalog_remove_herf = ''.join(url_list)
 991         response = self.perform_request(req_type='DELETE',
 992                                         url=catalog_remove_herf,
 993                                         headers=headers)
 994
 995         if response.status_code == requests.codes.no_content:
 996             self.logger.debug("Deleted Catalog {}".format(image_id))
 997             return image_id
 998         else:
 999             raise vimconn.vimconnException("Fail to delete Catalog {}".format(image_id))
1000
1001
1002     def catalog_exists(self, catalog_name, catalogs):
1003         """
1004
1005         :param catalog_name:
1006         :param catalogs:
1007         :return:
1008         """
1009         for catalog in catalogs:
1010             if catalog['name'] == catalog_name:
1011                 return True
1012         return False
1013
1014     def create_vimcatalog(self, vca=None, catalog_name=None):
1015         """ Create new catalog entry in vCloud director.
1016
1017             Args
1018                 vca:  vCloud director.
1019                 catalog_name catalog that client wish to create.   Note no validation done for a name.
1020                 Client must make sure that provide valid string representation.
1021
1022              Return (bool) True if catalog created.
1023
1024         """
1025         try:
1026             result = vca.create_catalog(catalog_name, catalog_name)
1027             if result is not None:
1028                 return True
1029             catalogs = vca.list_catalogs()
1030         except:
1031             return False
1032         return self.catalog_exists(catalog_name, catalogs)
1033
1034     # noinspection PyIncorrectDocstring
1035     def upload_ovf(self, vca=None, catalog_name=None, image_name=None, media_file_name=None,
1036                    description='', progress=False, chunk_bytes=128 * 1024):
1037         """
1038         Uploads a OVF file to a vCloud catalog
1039
1040         :param chunk_bytes:
1041         :param progress:
1042         :param description:
1043         :param image_name:
1044         :param vca:
1045         :param catalog_name: (str): The name of the catalog to upload the media.
1046         :param media_file_name: (str): The name of the local media file to upload.
1047         :return: (bool) True if the media file was successfully uploaded, false otherwise.
1048         """
1049         os.path.isfile(media_file_name)
1050         statinfo = os.stat(media_file_name)
1051
1052         #  find a catalog entry where we upload OVF.
1053         #  create vApp Template and check the status if vCD able to read OVF it will respond with appropirate
1054         #  status change.
1055         #  if VCD can parse OVF we upload VMDK file
1056         try:
1057             for catalog in vca.list_catalogs():
1058                 if catalog_name != catalog['name']:
1059                     continue
1060                 catalog_href = "{}/api/catalog/{}/action/upload".format(self.url, catalog['id'])
1061                 data = """
1062                 <UploadVAppTemplateParams name="{}" xmlns="http://www.vmware.com/vcloud/v1.5" xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/1"><Description>{} vApp Template</Description></UploadVAppTemplateParams>
1063                 """.format(catalog_name, description)
1064
1065                 if self.client:
1066                     headers = {'Accept':'application/*+xml;version=' + API_VERSION,
1067                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
1068                     headers['Content-Type'] = 'application/vnd.vmware.vcloud.uploadVAppTemplateParams+xml'
1069
1070                 response = self.perform_request(req_type='POST',
1071                                                 url=catalog_href,
1072                                                 headers=headers,
1073                                                 data=data)
1074
1075                 if response.status_code == requests.codes.created:
1076                     catalogItem = XmlElementTree.fromstring(response.content)
1077                     entity = [child for child in catalogItem if
1078                               child.get("type") == "application/vnd.vmware.vcloud.vAppTemplate+xml"][0]
1079                     href = entity.get('href')
1080                     template = href
1081
1082                     response = self.perform_request(req_type='GET',
1083                                                     url=href,
1084                                                     headers=headers)
1085
1086                     if response.status_code == requests.codes.ok:
1087                         headers['Content-Type'] = 'Content-Type text/xml'
1088                         result = re.search('rel="upload:default"\shref="(.*?\/descriptor.ovf)"',response.content)
1089                         if result:
1090                             transfer_href = result.group(1)
1091
1092                         response = self.perform_request(req_type='PUT',
1093                                                     url=transfer_href,
1094                                                     headers=headers,
1095                                                     data=open(media_file_name, 'rb'))
1096                         if response.status_code != requests.codes.ok:
1097                             self.logger.debug(
1098                                 "Failed create vApp template for catalog name {} and image {}".format(catalog_name,
1099                                                                                                       media_file_name))
1100                             return False
1101
1102                     # TODO fix this with aync block
1103                     time.sleep(5)
1104
1105                     self.logger.debug("vApp template for catalog name {} and image {}".format(catalog_name, media_file_name))
1106
1107                     # uploading VMDK file
1108                     # check status of OVF upload and upload remaining files.
1109                     response = self.perform_request(req_type='GET',
1110                                                     url=template,
1111                                                     headers=headers)
1112
1113                     if response.status_code == requests.codes.ok:
1114                         result = re.search('rel="upload:default"\s*href="(.*?vmdk)"',response.content)
1115                         if result:
1116                             link_href = result.group(1)
1117                         # we skip ovf since it already uploaded.
1118                         if 'ovf' in link_href:
1119                             continue
1120                         # The OVF file and VMDK must be in a same directory
1121                         head, tail = os.path.split(media_file_name)
1122                         file_vmdk = head + '/' + link_href.split("/")[-1]
1123                         if not os.path.isfile(file_vmdk):
1124                             return False
1125                         statinfo = os.stat(file_vmdk)
1126                         if statinfo.st_size == 0:
1127                             return False
1128                         hrefvmdk = link_href
1129
1130                         if progress:
1131                             widgets = ['Uploading file: ', Percentage(), ' ', Bar(), ' ', ETA(), ' ',
1132                                            FileTransferSpeed()]
1133                             progress_bar = ProgressBar(widgets=widgets, maxval=statinfo.st_size).start()
1134
1135                         bytes_transferred = 0
1136                         f = open(file_vmdk, 'rb')
1137                         while bytes_transferred < statinfo.st_size:
1138                             my_bytes = f.read(chunk_bytes)
1139                             if len(my_bytes) <= chunk_bytes:
1140                                 headers['Content-Range'] = 'bytes %s-%s/%s' % (
1141                                     bytes_transferred, len(my_bytes) - 1, statinfo.st_size)
1142                                 headers['Content-Length'] = str(len(my_bytes))
1143                                 response = requests.put(url=hrefvmdk,
1144                                                          headers=headers,
1145                                                          data=my_bytes,
1146                                                          verify=False)
1147                                 if response.status_code == requests.codes.ok:
1148                                     bytes_transferred += len(my_bytes)
1149                                     if progress:
1150                                         progress_bar.update(bytes_transferred)
1151                                 else:
1152                                     self.logger.debug(
1153                                         'file upload failed with error: [%s] %s' % (response.status_code,
1154                                                                                         response.content))
1155
1156                                     f.close()
1157                                     return False
1158                         f.close()
1159                         if progress:
1160                             progress_bar.finish()
1161                             time.sleep(10)
1162                     return True
1163                 else:
1164                     self.logger.debug("Failed retrieve vApp template for catalog name {} for OVF {}".
1165                                       format(catalog_name, media_file_name))
1166                     return False
1167         except Exception as exp:
1168             self.logger.debug("Failed while uploading OVF to catalog {} for OVF file {} with Exception {}"
1169                 .format(catalog_name,media_file_name, exp))
1170             raise vimconn.vimconnException(
1171                 "Failed while uploading OVF to catalog {} for OVF file {} with Exception {}"
1172                 .format(catalog_name,media_file_name, exp))
1173
1174         self.logger.debug("Failed retrieve catalog name {} for OVF file {}".format(catalog_name, media_file_name))
1175         return False
1176
1177     def upload_vimimage(self, vca=None, catalog_name=None, media_name=None, medial_file_name=None, progress=False):
1178         """Upload media file"""
1179         # TODO add named parameters for readability
1180
1181         return self.upload_ovf(vca=vca, catalog_name=catalog_name, image_name=media_name.split(".")[0],
1182                                media_file_name=medial_file_name, description='medial_file_name', progress=progress)
1183
1184     def validate_uuid4(self, uuid_string=None):
1185         """  Method validate correct format of UUID.
1186
1187         Return: true if string represent valid uuid
1188         """
1189         try:
1190             val = uuid.UUID(uuid_string, version=4)
1191         except ValueError:
1192             return False
1193         return True
1194
1195     def get_catalogid(self, catalog_name=None, catalogs=None):
1196         """  Method check catalog and return catalog ID in UUID format.
1197
1198         Args
1199             catalog_name: catalog name as string
1200             catalogs:  list of catalogs.
1201
1202         Return: catalogs uuid
1203         """
1204
1205         for catalog in catalogs:
1206             if catalog['name'] == catalog_name:
1207                 catalog_id = catalog['id']
1208                 return catalog_id
1209         return None
1210
1211     def get_catalogbyid(self, catalog_uuid=None, catalogs=None):
1212         """  Method check catalog and return catalog name lookup done by catalog UUID.
1213
1214         Args
1215             catalog_name: catalog name as string
1216             catalogs:  list of catalogs.
1217
1218         Return: catalogs name or None
1219         """
1220
1221         if not self.validate_uuid4(uuid_string=catalog_uuid):
1222             return None
1223
1224         for catalog in catalogs:
1225             catalog_id = catalog.get('id')
1226             if catalog_id == catalog_uuid:
1227                 return catalog.get('name')
1228         return None
1229
1230     def get_catalog_obj(self, catalog_uuid=None, catalogs=None):
1231         """  Method check catalog and return catalog name lookup done by catalog UUID.
1232
1233         Args
1234             catalog_name: catalog name as string
1235             catalogs:  list of catalogs.
1236
1237         Return: catalogs name or None
1238         """
1239
1240         if not self.validate_uuid4(uuid_string=catalog_uuid):
1241             return None
1242
1243         for catalog in catalogs:
1244             catalog_id = catalog.get('id')
1245             if catalog_id == catalog_uuid:
1246                 return catalog
1247         return None
1248
1249     def get_image_id_from_path(self, path=None, progress=False):
1250         """  Method upload OVF image to vCloud director.
1251
1252         Each OVF image represented as single catalog entry in vcloud director.
1253         The method check for existing catalog entry.  The check done by file name without file extension.
1254
1255         if given catalog name already present method will respond with existing catalog uuid otherwise
1256         it will create new catalog entry and upload OVF file to newly created catalog.
1257
1258         If method can't create catalog entry or upload a file it will throw exception.
1259
1260         Method accept boolean flag progress that will output progress bar. It useful method
1261         for standalone upload use case. In case to test large file upload.
1262
1263         Args
1264             path: - valid path to OVF file.
1265             progress - boolean progress bar show progress bar.
1266
1267         Return: if image uploaded correct method will provide image catalog UUID.
1268         """
1269
1270         if not path:
1271             raise vimconn.vimconnException("Image path can't be None.")
1272
1273         if not os.path.isfile(path):
1274             raise vimconn.vimconnException("Can't read file. File not found.")
1275
1276         if not os.access(path, os.R_OK):
1277             raise vimconn.vimconnException("Can't read file. Check file permission to read.")
1278
1279         self.logger.debug("get_image_id_from_path() client requesting {} ".format(path))
1280
1281         dirpath, filename = os.path.split(path)
1282         flname, file_extension = os.path.splitext(path)
1283         if file_extension != '.ovf':
1284             self.logger.debug("Wrong file extension {} connector support only OVF container.".format(file_extension))
1285             raise vimconn.vimconnException("Wrong container.  vCloud director supports only OVF.")
1286
1287         catalog_name = os.path.splitext(filename)[0]
1288         catalog_md5_name = hashlib.md5(path).hexdigest()
1289         self.logger.debug("File name {} Catalog Name {} file path {} "
1290                           "vdc catalog name {}".format(filename, catalog_name, path, catalog_md5_name))
1291
1292         try:
1293             org,vdc = self.get_vdc_details()
1294             catalogs = org.list_catalogs()
1295         except Exception as exp:
1296             self.logger.debug("Failed get catalogs() with Exception {} ".format(exp))
1297             raise vimconn.vimconnException("Failed get catalogs() with Exception {} ".format(exp))
1298
1299         if len(catalogs) == 0:
1300             self.logger.info("Creating a new catalog entry {} in vcloud director".format(catalog_name))
1301             result = self.create_vimcatalog(org, catalog_md5_name)
1302             if not result:
1303                 raise vimconn.vimconnException("Failed create new catalog {} ".format(catalog_md5_name))
1304
1305             result = self.upload_vimimage(vca=org, catalog_name=catalog_md5_name,
1306                                           media_name=filename, medial_file_name=path, progress=progress)
1307             if not result:
1308                 raise vimconn.vimconnException("Failed create vApp template for catalog {} ".format(catalog_name))
1309             return self.get_catalogid(catalog_name, catalogs)
1310         else:
1311             for catalog in catalogs:
1312                 # search for existing catalog if we find same name we return ID
1313                 # TODO optimize this
1314                 if catalog['name'] == catalog_md5_name:
1315                     self.logger.debug("Found existing catalog entry for {} "
1316                                       "catalog id {}".format(catalog_name,
1317                                                              self.get_catalogid(catalog_md5_name, catalogs)))
1318                     return self.get_catalogid(catalog_md5_name, catalogs)
1319
1320         # if we didn't find existing catalog we create a new one and upload image.
1321         self.logger.debug("Creating new catalog entry {} - {}".format(catalog_name, catalog_md5_name))
1322         result = self.create_vimcatalog(org, catalog_md5_name)
1323         if not result:
1324             raise vimconn.vimconnException("Failed create new catalog {} ".format(catalog_md5_name))
1325
1326         result = self.upload_vimimage(vca=org, catalog_name=catalog_md5_name,
1327                                       media_name=filename, medial_file_name=path, progress=progress)
1328         if not result:
1329             raise vimconn.vimconnException("Failed create vApp template for catalog {} ".format(catalog_md5_name))
1330
1331         return self.get_catalogid(catalog_md5_name, org.list_catalogs())
1332
1333     def get_image_list(self, filter_dict={}):
1334         '''Obtain tenant images from VIM
1335         Filter_dict can be:
1336             name: image name
1337             id: image uuid
1338             checksum: image checksum
1339             location: image path
1340         Returns the image list of dictionaries:
1341             [{<the fields at Filter_dict plus some VIM specific>}, ...]
1342             List can be empty
1343         '''
1344
1345         try:
1346             org, vdc = self.get_vdc_details()
1347             image_list = []
1348             catalogs = org.list_catalogs()
1349             if len(catalogs) == 0:
1350                 return image_list
1351             else:
1352                 for catalog in catalogs:
1353                     catalog_uuid = catalog.get('id')
1354                     name = catalog.get('name')
1355                     filtered_dict = {}
1356                     if filter_dict.get("name") and filter_dict["name"] != name:
1357                         continue
1358                     if filter_dict.get("id") and filter_dict["id"] != catalog_uuid:
1359                         continue
1360                     filtered_dict ["name"] = name
1361                     filtered_dict ["id"] = catalog_uuid
1362                     image_list.append(filtered_dict)
1363
1364                 self.logger.debug("List of already created catalog items: {}".format(image_list))
1365                 return image_list
1366         except Exception as exp:
1367             raise vimconn.vimconnException("Exception occured while retriving catalog items {}".format(exp))
1368
1369     def get_vappid(self, vdc=None, vapp_name=None):
1370         """ Method takes vdc object and vApp name and returns vapp uuid or None
1371
1372         Args:
1373             vdc: The VDC object.
1374             vapp_name: is application vappp name identifier
1375
1376         Returns:
1377                 The return vApp name otherwise None
1378         """
1379         if vdc is None or vapp_name is None:
1380             return None
1381         # UUID has following format https://host/api/vApp/vapp-30da58a3-e7c7-4d09-8f68-d4c8201169cf
1382         try:
1383             refs = filter(lambda ref: ref.name == vapp_name and ref.type_ == 'application/vnd.vmware.vcloud.vApp+xml',
1384                           vdc.ResourceEntities.ResourceEntity)
1385             #For python3
1386             #refs = [ref for ref in vdc.ResourceEntities.ResourceEntity\
1387             #         if ref.name == vapp_name and ref.type_ == 'application/vnd.vmware.vcloud.vApp+xml']
1388             if len(refs) == 1:
1389                 return refs[0].href.split("vapp")[1][1:]
1390         except Exception as e:
1391             self.logger.exception(e)
1392             return False
1393         return None
1394
1395     def check_vapp(self, vdc=None, vapp_uuid=None):
1396         """ Method Method returns True or False if vapp deployed in vCloud director
1397
1398             Args:
1399                 vca: Connector to VCA
1400                 vdc: The VDC object.
1401                 vappid: vappid is application identifier
1402
1403             Returns:
1404                 The return True if vApp deployed
1405                 :param vdc:
1406                 :param vapp_uuid:
1407         """
1408         try:
1409             refs = filter(lambda ref:
1410                           ref.type_ == 'application/vnd.vmware.vcloud.vApp+xml',
1411                           vdc.ResourceEntities.ResourceEntity)
1412             #For python3
1413             #refs = [ref for ref in vdc.ResourceEntities.ResourceEntity\
1414             #         if ref.type_ == 'application/vnd.vmware.vcloud.vApp+xml']
1415             for ref in refs:
1416                 vappid = ref.href.split("vapp")[1][1:]
1417                 # find vapp with respected vapp uuid
1418                 if vappid == vapp_uuid:
1419                     return True
1420         except Exception as e:
1421             self.logger.exception(e)
1422             return False
1423         return False
1424
1425     def get_namebyvappid(self, vapp_uuid=None):
1426         """Method returns vApp name from vCD and lookup done by vapp_id.
1427
1428         Args:
1429             vapp_uuid: vappid is application identifier
1430
1431         Returns:
1432             The return vApp name otherwise None
1433         """
1434         try:
1435             if self.client and vapp_uuid:
1436                 vapp_call = "{}/api/vApp/vapp-{}".format(self.url, vapp_uuid)
1437                 headers = {'Accept':'application/*+xml;version=' + API_VERSION,
1438                      'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
1439
1440                 response = self.perform_request(req_type='GET',
1441                                                 url=vapp_call,
1442                                                 headers=headers)
1443                 #Retry login if session expired & retry sending request
1444                 if response.status_code == 403:
1445                     response = self.retry_rest('GET', vapp_call)
1446
1447                 tree = XmlElementTree.fromstring(response.content)
1448                 return tree.attrib['name']
1449         except Exception as e:
1450             self.logger.exception(e)
1451             return None
1452         return None
1453
1454     def new_vminstance(self, name=None, description="", start=False, image_id=None, flavor_id=None, net_list=[],
1455                        cloud_config=None, disk_list=None, availability_zone_index=None, availability_zone_list=None):
1456         """Adds a VM instance to VIM
1457         Params:
1458             'start': (boolean) indicates if VM must start or created in pause mode.
1459             'image_id','flavor_id': image and flavor VIM id to use for the VM
1460             'net_list': list of interfaces, each one is a dictionary with:
1461                 'name': (optional) name for the interface.
1462                 'net_id': VIM network id where this interface must be connect to. Mandatory for type==virtual
1463                 'vpci': (optional) virtual vPCI address to assign at the VM. Can be ignored depending on VIM capabilities
1464                 'model': (optional and only have sense for type==virtual) interface model: virtio, e2000, ...
1465                 'mac_address': (optional) mac address to assign to this interface
1466                 #TODO: CHECK if an optional 'vlan' parameter is needed for VIMs when type if VF and net_id is not provided,
1467                     the VLAN tag to be used. In case net_id is provided, the internal network vlan is used for tagging VF
1468                 'type': (mandatory) can be one of:
1469                     'virtual', in this case always connected to a network of type 'net_type=bridge'
1470                      'PCI-PASSTHROUGH' or 'PF' (passthrough): depending on VIM capabilities it can be connected to a data/ptp network ot it
1471                            can created unconnected
1472                      'SR-IOV' or 'VF' (SRIOV with VLAN tag): same as PF for network connectivity.
1473                      'VFnotShared'(SRIOV without VLAN tag) same as PF for network connectivity. VF where no other VFs
1474                             are allocated on the same physical NIC
1475                 'bw': (optional) only for PF/VF/VFnotShared. Minimal Bandwidth required for the interface in GBPS
1476                 'port_security': (optional) If False it must avoid any traffic filtering at this interface. If missing
1477                                 or True, it must apply the default VIM behaviour
1478                 After execution the method will add the key:
1479                 'vim_id': must be filled/added by this method with the VIM identifier generated by the VIM for this
1480                         interface. 'net_list' is modified
1481             'cloud_config': (optional) dictionary with:
1482                 'key-pairs': (optional) list of strings with the public key to be inserted to the default user
1483                 'users': (optional) list of users to be inserted, each item is a dict with:
1484                     'name': (mandatory) user name,
1485                     'key-pairs': (optional) list of strings with the public key to be inserted to the user
1486                 'user-data': (optional) can be a string with the text script to be passed directly to cloud-init,
1487                     or a list of strings, each one contains a script to be passed, usually with a MIMEmultipart file
1488                 'config-files': (optional). List of files to be transferred. Each item is a dict with:
1489                     'dest': (mandatory) string with the destination absolute path
1490                     'encoding': (optional, by default text). Can be one of:
1491                         'b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64'
1492                     'content' (mandatory): string with the content of the file
1493                     'permissions': (optional) string with file permissions, typically octal notation '0644'
1494                     'owner': (optional) file owner, string with the format 'owner:group'
1495                 'boot-data-drive': boolean to indicate if user-data must be passed using a boot drive (hard disk)
1496             'disk_list': (optional) list with additional disks to the VM. Each item is a dict with:
1497                 'image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted
1498                 'size': (mandatory) string with the size of the disk in GB
1499             availability_zone_index: Index of availability_zone_list to use for this this VM. None if not AV required
1500             availability_zone_list: list of availability zones given by user in the VNFD descriptor.  Ignore if
1501                 availability_zone_index is None
1502         Returns a tuple with the instance identifier and created_items or raises an exception on error
1503             created_items can be None or a dictionary where this method can include key-values that will be passed to
1504             the method delete_vminstance and action_vminstance. Can be used to store created ports, volumes, etc.
1505             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
1506             as not present.
1507         """
1508         self.logger.info("Creating new instance for entry {}".format(name))
1509         self.logger.debug("desc {} boot {} image_id: {} flavor_id: {} net_list: {} cloud_config {} disk_list {}".format(
1510                                     description, start, image_id, flavor_id, net_list, cloud_config, disk_list))
1511
1512         #new vm name = vmname + tenant_id + uuid
1513         new_vm_name = [name, '-', str(uuid.uuid4())]
1514         vmname_andid = ''.join(new_vm_name)
1515
1516         for net in net_list:
1517             if net['type'] == "SR-IOV" or net['type'] == "PCI-PASSTHROUGH":
1518                 raise vimconn.vimconnNotSupportedException(
1519                       "Current vCD version does not support type : {}".format(net['type']))
1520
1521         if len(net_list) > 10:
1522             raise vimconn.vimconnNotSupportedException(
1523                       "The VM hardware versions 7 and above support upto 10 NICs only")
1524
1525         # if vm already deployed we return existing uuid
1526         # we check for presence of VDC, Catalog entry and Flavor.
1527         org, vdc = self.get_vdc_details()
1528         if vdc is None:
1529             raise vimconn.vimconnNotFoundException(
1530                 "new_vminstance(): Failed create vApp {}: (Failed retrieve VDC information)".format(name))
1531         catalogs = org.list_catalogs()
1532         if catalogs is None:
1533             #Retry once, if failed by refreshing token
1534             self.get_token()
1535             org = Org(self.client, resource=self.client.get_org())
1536             catalogs = org.list_catalogs()
1537         if catalogs is None:
1538             raise vimconn.vimconnNotFoundException(
1539                 "new_vminstance(): Failed create vApp {}: (Failed retrieve catalogs list)".format(name))
1540
1541         catalog_hash_name = self.get_catalogbyid(catalog_uuid=image_id, catalogs=catalogs)
1542         if catalog_hash_name:
1543             self.logger.info("Found catalog entry {} for image id {}".format(catalog_hash_name, image_id))
1544         else:
1545             raise vimconn.vimconnNotFoundException("new_vminstance(): Failed create vApp {}: "
1546                                                    "(Failed retrieve catalog information {})".format(name, image_id))
1547
1548
1549         # Set vCPU and Memory based on flavor.
1550         vm_cpus = None
1551         vm_memory = None
1552         vm_disk = None
1553         numas = None
1554
1555         if flavor_id is not None:
1556             if flavor_id not in vimconnector.flavorlist:
1557                 raise vimconn.vimconnNotFoundException("new_vminstance(): Failed create vApp {}: "
1558                                                        "Failed retrieve flavor information "
1559                                                        "flavor id {}".format(name, flavor_id))
1560             else:
1561                 try:
1562                     flavor = vimconnector.flavorlist[flavor_id]
1563                     vm_cpus = flavor[FLAVOR_VCPUS_KEY]
1564                     vm_memory = flavor[FLAVOR_RAM_KEY]
1565                     vm_disk = flavor[FLAVOR_DISK_KEY]
1566                     extended = flavor.get("extended", None)
1567                     if extended:
1568                         numas=extended.get("numas", None)
1569
1570                 except Exception as exp:
1571                     raise vimconn.vimconnException("Corrupted flavor. {}.Exception: {}".format(flavor_id, exp))
1572
1573         # image upload creates template name as catalog name space Template.
1574         templateName = self.get_catalogbyid(catalog_uuid=image_id, catalogs=catalogs)
1575         power_on = 'false'
1576         if start:
1577             power_on = 'true'
1578
1579         # client must provide at least one entry in net_list if not we report error
1580         #If net type is mgmt, then configure it as primary net & use its NIC index as primary NIC
1581         #If no mgmt, then the 1st NN in netlist is considered as primary net.
1582         primary_net = None
1583         primary_netname = None
1584         network_mode = 'bridged'
1585         if net_list is not None and len(net_list) > 0:
1586             for net in net_list:
1587                 if 'use' in net and net['use'] == 'mgmt' and not primary_net:
1588                     primary_net = net
1589             if primary_net is None:
1590                 primary_net = net_list[0]
1591
1592             try:
1593                 primary_net_id = primary_net['net_id']
1594                 network_dict = self.get_vcd_network(network_uuid=primary_net_id)
1595                 if 'name' in network_dict:
1596                     primary_netname = network_dict['name']
1597
1598             except KeyError:
1599                 raise vimconn.vimconnException("Corrupted flavor. {}".format(primary_net))
1600         else:
1601             raise vimconn.vimconnUnexpectedResponse("new_vminstance(): Failed network list is empty.".format(name))
1602
1603         # use: 'data', 'bridge', 'mgmt'
1604         # create vApp.  Set vcpu and ram based on flavor id.
1605         try:
1606             vdc_obj = VDC(self.client, resource=org.get_vdc(self.tenant_name))
1607             if not vdc_obj:
1608                 raise vimconn.vimconnNotFoundException("new_vminstance(): Failed to get VDC object")
1609
1610             for retry in (1,2):
1611                 items = org.get_catalog_item(catalog_hash_name, catalog_hash_name)
1612                 catalog_items = [items.attrib]
1613
1614                 if len(catalog_items) == 1:
1615                     if self.client:
1616                         headers = {'Accept':'application/*+xml;version=' + API_VERSION,
1617                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
1618
1619                     response = self.perform_request(req_type='GET',
1620                                                 url=catalog_items[0].get('href'),
1621                                                 headers=headers)
1622                     catalogItem = XmlElementTree.fromstring(response.content)
1623                     entity = [child for child in catalogItem if child.get("type") == "application/vnd.vmware.vcloud.vAppTemplate+xml"][0]
1624                     vapp_tempalte_href = entity.get("href")
1625
1626                 response = self.perform_request(req_type='GET',
1627                                                     url=vapp_tempalte_href,
1628                                                     headers=headers)
1629                 if response.status_code != requests.codes.ok:
1630                     self.logger.debug("REST API call {} failed. Return status code {}".format(vapp_tempalte_href,
1631                                                                                            response.status_code))
1632                 else:
1633                     result = (response.content).replace("\n"," ")
1634
1635                 src = re.search('<Vm goldMaster="false"\sstatus="\d+"\sname="(.*?)"\s'
1636                                                'id="(\w+:\w+:vm:.*?)"\shref="(.*?)"\s'
1637                               'type="application/vnd\.vmware\.vcloud\.vm\+xml',result)
1638                 if src:
1639                     vm_name = src.group(1)
1640                     vm_id = src.group(2)
1641                     vm_href = src.group(3)
1642
1643                 cpus = re.search('<rasd:Description>Number of Virtual CPUs</.*?>(\d+)</rasd:VirtualQuantity>',result).group(1)
1644                 memory_mb = re.search('<rasd:Description>Memory Size</.*?>(\d+)</rasd:VirtualQuantity>',result).group(1)
1645                 cores = re.search('<vmw:CoresPerSocket ovf:required.*?>(\d+)</vmw:CoresPerSocket>',result).group(1)
1646
1647                 headers['Content-Type'] = 'application/vnd.vmware.vcloud.instantiateVAppTemplateParams+xml'
1648                 vdc_id = vdc.get('id').split(':')[-1]
1649                 instantiate_vapp_href = "{}/api/vdc/{}/action/instantiateVAppTemplate".format(self.url,
1650                                                                                                 vdc_id)
1651                 data = """<?xml version="1.0" encoding="UTF-8"?>
1652                 <InstantiateVAppTemplateParams
1653                 xmlns="http://www.vmware.com/vcloud/v1.5"
1654                 name="{}"
1655                 deploy="false"
1656                 powerOn="false"
1657                 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
1658                 xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/1">
1659                 <Description>Vapp instantiation</Description>
1660                 <InstantiationParams>
1661                      <NetworkConfigSection>
1662                          <ovf:Info>Configuration parameters for logical networks</ovf:Info>
1663                          <NetworkConfig networkName="None">
1664                              <Configuration>
1665                                  <ParentNetwork href=""/>
1666                                  <FenceMode>bridged</FenceMode>
1667                              </Configuration>
1668                          </NetworkConfig>
1669                      </NetworkConfigSection>
1670                 <LeaseSettingsSection
1671                 type="application/vnd.vmware.vcloud.leaseSettingsSection+xml">
1672                 <ovf:Info>Lease Settings</ovf:Info>
1673                 <StorageLeaseInSeconds>172800</StorageLeaseInSeconds>
1674                 <StorageLeaseExpiration>2014-04-25T08:08:16.438-07:00</StorageLeaseExpiration>
1675                 </LeaseSettingsSection>
1676                 </InstantiationParams>
1677                 <Source href="{}"/>
1678                 <SourcedItem>
1679                 <Source href="{}" id="{}" name="{}"
1680                 type="application/vnd.vmware.vcloud.vm+xml"/>
1681                 <VmGeneralParams>
1682                     <NeedsCustomization>false</NeedsCustomization>
1683                 </VmGeneralParams>
1684                 <InstantiationParams>
1685                       <NetworkConnectionSection>
1686                       <ovf:Info>Specifies the available VM network connections</ovf:Info>
1687                       <NetworkConnection network="{}">
1688                       <NetworkConnectionIndex>0</NetworkConnectionIndex>
1689                       <IsConnected>true</IsConnected>
1690                       <IpAddressAllocationMode>DHCP</IpAddressAllocationMode>
1691                       </NetworkConnection>
1692                       </NetworkConnectionSection><ovf:VirtualHardwareSection>
1693                       <ovf:Info>Virtual hardware requirements</ovf:Info>
1694                       <ovf:Item xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData"
1695                       xmlns:vmw="http://www.vmware.com/schema/ovf">
1696                       <rasd:AllocationUnits>hertz * 10^6</rasd:AllocationUnits>
1697                       <rasd:Description>Number of Virtual CPUs</rasd:Description>
1698                       <rasd:ElementName xmlns:py="http://codespeak.net/lxml/objectify/pytype" py:pytype="str">{cpu} virtual CPU(s)</rasd:ElementName>
1699                       <rasd:InstanceID>4</rasd:InstanceID>
1700                       <rasd:Reservation>0</rasd:Reservation>
1701                       <rasd:ResourceType>3</rasd:ResourceType>
1702                       <rasd:VirtualQuantity xmlns:py="http://codespeak.net/lxml/objectify/pytype" py:pytype="int">{cpu}</rasd:VirtualQuantity>
1703                       <rasd:Weight>0</rasd:Weight>
1704                       <vmw:CoresPerSocket ovf:required="false">{core}</vmw:CoresPerSocket>
1705                       </ovf:Item><ovf:Item xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData">
1706                       <rasd:AllocationUnits>byte * 2^20</rasd:AllocationUnits>
1707                       <rasd:Description>Memory Size</rasd:Description>
1708                       <rasd:ElementName xmlns:py="http://codespeak.net/lxml/objectify/pytype" py:pytype="str">{memory} MB of memory</rasd:ElementName>
1709                       <rasd:InstanceID>5</rasd:InstanceID>
1710                       <rasd:Reservation>0</rasd:Reservation>
1711                       <rasd:ResourceType>4</rasd:ResourceType>
1712                       <rasd:VirtualQuantity xmlns:py="http://codespeak.net/lxml/objectify/pytype" py:pytype="int">{memory}</rasd:VirtualQuantity>
1713                       <rasd:Weight>0</rasd:Weight>
1714                       </ovf:Item>
1715                 </ovf:VirtualHardwareSection>
1716                 </InstantiationParams>
1717                 </SourcedItem>
1718                 <AllEULAsAccepted>false</AllEULAsAccepted>
1719                 </InstantiateVAppTemplateParams>""".format(vmname_andid,
1720                                                      vapp_tempalte_href,
1721                                                                 vm_href,
1722                                                                   vm_id,
1723                                                                 vm_name,
1724                                                         primary_netname,
1725                                                                cpu=cpus,
1726                                                              core=cores,
1727                                                        memory=memory_mb)
1728
1729                 response = self.perform_request(req_type='POST',
1730                                                 url=instantiate_vapp_href,
1731                                                 headers=headers,
1732                                                 data=data)
1733
1734                 if response.status_code != 201:
1735                     self.logger.error("REST call {} failed reason : {}"\
1736                          "status code : {}".format(instantiate_vapp_href,
1737                                                         response.content,
1738                                                    response.status_code))
1739                     raise vimconn.vimconnException("new_vminstance(): Failed to create"\
1740                                                         "vAapp {}".format(vmname_andid))
1741                 else:
1742                     vapptask = self.get_task_from_response(response.content)
1743
1744                 if vapptask is None and retry==1:
1745                     self.get_token() # Retry getting token
1746                     continue
1747                 else:
1748                     break
1749
1750             if vapptask is None or vapptask is False:
1751                 raise vimconn.vimconnUnexpectedResponse(
1752                     "new_vminstance(): failed to create vApp {}".format(vmname_andid))
1753
1754             # wait for task to complete
1755             result = self.client.get_task_monitor().wait_for_success(task=vapptask)
1756
1757             if result.get('status') == 'success':
1758                 self.logger.debug("new_vminstance(): Sucessfully created Vapp {}".format(vmname_andid))
1759             else:
1760                 raise vimconn.vimconnUnexpectedResponse(
1761                     "new_vminstance(): failed to create vApp {}".format(vmname_andid))
1762
1763         except Exception as exp:
1764             raise vimconn.vimconnUnexpectedResponse(
1765                 "new_vminstance(): failed to create vApp {} with Exception:{}".format(vmname_andid, exp))
1766
1767         # we should have now vapp in undeployed state.
1768         try:
1769             vdc_obj = VDC(self.client, href=vdc.get('href'))
1770             vapp_resource = vdc_obj.get_vapp(vmname_andid)
1771             vapp_uuid = vapp_resource.get('id').split(':')[-1]
1772             vapp = VApp(self.client, resource=vapp_resource)
1773
1774         except Exception as exp:
1775             raise vimconn.vimconnUnexpectedResponse(
1776                     "new_vminstance(): Failed to retrieve vApp {} after creation: Exception:{}"
1777                     .format(vmname_andid, exp))
1778
1779         if vapp_uuid is None:
1780             raise vimconn.vimconnUnexpectedResponse(
1781                 "new_vminstance(): Failed to retrieve vApp {} after creation".format(
1782                                                                             vmname_andid))
1783
1784         #Add PCI passthrough/SRIOV configrations
1785         vm_obj = None
1786         pci_devices_info = []
1787         sriov_net_info = []
1788         reserve_memory = False
1789
1790         for net in net_list:
1791             if net["type"] == "PF" or net["type"] == "PCI-PASSTHROUGH":
1792                 pci_devices_info.append(net)
1793             elif (net["type"] == "VF" or net["type"] == "SR-IOV" or net["type"] == "VFnotShared") and 'net_id'in net:
1794                 sriov_net_info.append(net)
1795
1796         #Add PCI
1797         if len(pci_devices_info) > 0:
1798             self.logger.info("Need to add PCI devices {} into VM {}".format(pci_devices_info,
1799                                                                         vmname_andid ))
1800             PCI_devices_status, vm_obj, vcenter_conect = self.add_pci_devices(vapp_uuid,
1801                                                                             pci_devices_info,
1802                                                                             vmname_andid)
1803             if PCI_devices_status:
1804                 self.logger.info("Added PCI devives {} to VM {}".format(
1805                                                             pci_devices_info,
1806                                                             vmname_andid)
1807                                  )
1808                 reserve_memory = True
1809             else:
1810                 self.logger.info("Fail to add PCI devives {} to VM {}".format(
1811                                                             pci_devices_info,
1812                                                             vmname_andid)
1813                                  )
1814
1815         # Modify vm disk
1816         if vm_disk:
1817             #Assuming there is only one disk in ovf and fast provisioning in organization vDC is disabled
1818             result = self.modify_vm_disk(vapp_uuid, vm_disk)
1819             if result :
1820                 self.logger.debug("Modified Disk size of VM {} ".format(vmname_andid))
1821
1822         #Add new or existing disks to vApp
1823         if disk_list:
1824             added_existing_disk = False
1825             for disk in disk_list:
1826                 if 'device_type' in disk and disk['device_type'] == 'cdrom':
1827                     image_id = disk['image_id']
1828                     # Adding CD-ROM to VM
1829                     # will revisit code once specification ready to support this feature
1830                     self.insert_media_to_vm(vapp, image_id)
1831                 elif "image_id" in disk and disk["image_id"] is not None:
1832                     self.logger.debug("Adding existing disk from image {} to vm {} ".format(
1833                                                                     disk["image_id"] , vapp_uuid))
1834                     self.add_existing_disk(catalogs=catalogs,
1835                                            image_id=disk["image_id"],
1836                                            size = disk["size"],
1837                                            template_name=templateName,
1838                                            vapp_uuid=vapp_uuid
1839                                            )
1840                     added_existing_disk = True
1841                 else:
1842                     #Wait till added existing disk gets reflected into vCD database/API
1843                     if added_existing_disk:
1844                         time.sleep(5)
1845                         added_existing_disk = False
1846                     self.add_new_disk(vapp_uuid, disk['size'])
1847
1848         if numas:
1849             # Assigning numa affinity setting
1850             for numa in numas:
1851                 if 'paired-threads-id' in numa:
1852                     paired_threads_id = numa['paired-threads-id']
1853                     self.set_numa_affinity(vapp_uuid, paired_threads_id)
1854
1855         # add NICs & connect to networks in netlist
1856         try:
1857             self.logger.info("Request to connect VM to a network: {}".format(net_list))
1858             nicIndex = 0
1859             primary_nic_index = 0
1860             for net in net_list:
1861                 # openmano uses network id in UUID format.
1862                 # vCloud Director need a name so we do reverse operation from provided UUID we lookup a name
1863                 # [{'use': 'bridge', 'net_id': '527d4bf7-566a-41e7-a9e7-ca3cdd9cef4f', 'type': 'virtual',
1864                 #   'vpci': '0000:00:11.0', 'name': 'eth0'}]
1865
1866                 if 'net_id' not in net:
1867                     continue
1868
1869                 #Using net_id as a vim_id i.e. vim interface id, as do not have saperate vim interface id
1870                 #Same will be returned in refresh_vms_status() as vim_interface_id
1871                 net['vim_id'] = net['net_id']  # Provide the same VIM identifier as the VIM network
1872
1873                 interface_net_id = net['net_id']
1874                 interface_net_name = self.get_network_name_by_id(network_uuid=interface_net_id)
1875                 interface_network_mode = net['use']
1876
1877                 if interface_network_mode == 'mgmt':
1878                     primary_nic_index = nicIndex
1879
1880                 """- POOL (A static IP address is allocated automatically from a pool of addresses.)
1881                                   - DHCP (The IP address is obtained from a DHCP service.)
1882                                   - MANUAL (The IP address is assigned manually in the IpAddress element.)
1883                                   - NONE (No IP addressing mode specified.)"""
1884
1885                 if primary_netname is not None:
1886                     nets = filter(lambda n: n.get('name') == interface_net_name, self.get_network_list())
1887                     #For python3
1888                     #nets = [n for n in self.get_network_list() if n.get('name') == interface_net_name]
1889                     if len(nets) == 1:
1890                         self.logger.info("new_vminstance(): Found requested network: {}".format(nets[0].get('name')))
1891
1892                         vdc_obj = VDC(self.client, href=vdc.get('href'))
1893                         vapp_resource = vdc_obj.get_vapp(vmname_andid)
1894                         vapp = VApp(self.client, resource=vapp_resource)
1895                         # connect network to VM - with all DHCP by default
1896                         task = vapp.connect_org_vdc_network(nets[0].get('name'))
1897
1898                         self.client.get_task_monitor().wait_for_success(task=task)
1899
1900                         type_list = ('PF', 'PCI-PASSTHROUGH', 'VF', 'SR-IOV', 'VFnotShared')
1901                         if 'type' in net and net['type'] not in type_list:
1902                             # fetching nic type from vnf
1903                             if 'model' in net:
1904                                 if net['model'] is not None and net['model'].lower() == 'virtio':
1905                                     nic_type = 'VMXNET3'
1906                                 else:
1907                                     nic_type = net['model']
1908
1909                                 self.logger.info("new_vminstance(): adding network adapter "\
1910                                                           "to a network {}".format(nets[0].get('name')))
1911                                 self.add_network_adapter_to_vms(vapp, nets[0].get('name'),
1912                                                                 primary_nic_index,
1913                                                                 nicIndex,
1914                                                                 net,
1915                                                                 nic_type=nic_type)
1916                             else:
1917                                 self.logger.info("new_vminstance(): adding network adapter "\
1918                                                          "to a network {}".format(nets[0].get('name')))
1919                                 self.add_network_adapter_to_vms(vapp, nets[0].get('name'),
1920                                                                 primary_nic_index,
1921                                                                 nicIndex,
1922                                                                 net)
1923                 nicIndex += 1
1924
1925             # cloud-init for ssh-key injection
1926             if cloud_config:
1927                 self.cloud_init(vapp,cloud_config)
1928
1929         # ############# Stub code for SRIOV #################
1930         #Add SRIOV
1931 #         if len(sriov_net_info) > 0:
1932 #             self.logger.info("Need to add SRIOV adapters {} into VM {}".format(sriov_net_info,
1933 #                                                                         vmname_andid ))
1934 #             sriov_status, vm_obj, vcenter_conect = self.add_sriov(vapp_uuid,
1935 #                                                                   sriov_net_info,
1936 #                                                                   vmname_andid)
1937 #             if sriov_status:
1938 #                 self.logger.info("Added SRIOV {} to VM {}".format(
1939 #                                                             sriov_net_info,
1940 #                                                             vmname_andid)
1941 #                                  )
1942 #                 reserve_memory = True
1943 #             else:
1944 #                 self.logger.info("Fail to add SRIOV {} to VM {}".format(
1945 #                                                             sriov_net_info,
1946 #                                                             vmname_andid)
1947 #                                  )
1948
1949             # If VM has PCI devices or SRIOV reserve memory for VM
1950             if reserve_memory:
1951                 memReserve = vm_obj.config.hardware.memoryMB
1952                 spec = vim.vm.ConfigSpec()
1953                 spec.memoryAllocation = vim.ResourceAllocationInfo(reservation=memReserve)
1954                 task = vm_obj.ReconfigVM_Task(spec=spec)
1955                 if task:
1956                     result = self.wait_for_vcenter_task(task, vcenter_conect)
1957                     self.logger.info("Reserved memory {} MB for "
1958                                      "VM VM status: {}".format(str(memReserve), result))
1959                 else:
1960                     self.logger.info("Fail to reserved memory {} to VM {}".format(
1961                                                                 str(memReserve), str(vm_obj)))
1962
1963             self.logger.debug("new_vminstance(): starting power on vApp {} ".format(vmname_andid))
1964
1965             vapp_id = vapp_resource.get('id').split(':')[-1]
1966             poweron_task = self.power_on_vapp(vapp_id, vmname_andid)
1967             result = self.client.get_task_monitor().wait_for_success(task=poweron_task)
1968             if result.get('status') == 'success':
1969                 self.logger.info("new_vminstance(): Successfully power on "\
1970                                              "vApp {}".format(vmname_andid))
1971             else:
1972                 self.logger.error("new_vminstance(): failed to power on vApp "\
1973                                                      "{}".format(vmname_andid))
1974
1975         except Exception as exp :
1976             # it might be a case if specific mandatory entry in dict is empty or some other pyVcloud exception
1977             self.logger.error("new_vminstance(): Failed create new vm instance {} with exception {}"
1978                               .format(name, exp))
1979             raise vimconn.vimconnException("new_vminstance(): Failed create new vm instance {} with exception {}"
1980                                            .format(name, exp))
1981
1982         # check if vApp deployed and if that the case return vApp UUID otherwise -1
1983         wait_time = 0
1984         vapp_uuid = None
1985         while wait_time <= MAX_WAIT_TIME:
1986             try:
1987                 vapp_resource = vdc_obj.get_vapp(vmname_andid)
1988                 vapp = VApp(self.client, resource=vapp_resource)
1989             except Exception as exp:
1990                 raise vimconn.vimconnUnexpectedResponse(
1991                         "new_vminstance(): Failed to retrieve vApp {} after creation: Exception:{}"
1992                         .format(vmname_andid, exp))
1993
1994             #if vapp and vapp.me.deployed:
1995             if vapp and vapp_resource.get('deployed') == 'true':
1996                 vapp_uuid = vapp_resource.get('id').split(':')[-1]
1997                 break
1998             else:
1999                 self.logger.debug("new_vminstance(): Wait for vApp {} to deploy".format(name))
2000                 time.sleep(INTERVAL_TIME)
2001
2002             wait_time +=INTERVAL_TIME
2003
2004         if vapp_uuid is not None:
2005             return vapp_uuid, None
2006         else:
2007             raise vimconn.vimconnUnexpectedResponse("new_vminstance(): Failed create new vm instance {}".format(name))
2008
2009     ##
2010     ##
2011     ##  based on current discussion
2012     ##
2013     ##
2014     ##  server:
2015     #   created: '2016-09-08T11:51:58'
2016     #   description: simple-instance.linux1.1
2017     #   flavor: ddc6776e-75a9-11e6-ad5f-0800273e724c
2018     #   hostId: e836c036-74e7-11e6-b249-0800273e724c
2019     #   image: dde30fe6-75a9-11e6-ad5f-0800273e724c
2020     #   status: ACTIVE
2021     #   error_msg:
2022     #   interfaces: …
2023     #
2024     def get_vminstance(self, vim_vm_uuid=None):
2025         """Returns the VM instance information from VIM"""
2026
2027         self.logger.debug("Client requesting vm instance {} ".format(vim_vm_uuid))
2028
2029         org, vdc = self.get_vdc_details()
2030         if vdc is None:
2031             raise vimconn.vimconnConnectionException(
2032                 "Failed to get a reference of VDC for a tenant {}".format(self.tenant_name))
2033
2034         vm_info_dict = self.get_vapp_details_rest(vapp_uuid=vim_vm_uuid)
2035         if not vm_info_dict:
2036             self.logger.debug("get_vminstance(): Failed to get vApp name by UUID {}".format(vim_vm_uuid))
2037             raise vimconn.vimconnNotFoundException("Failed to get vApp name by UUID {}".format(vim_vm_uuid))
2038
2039         status_key = vm_info_dict['status']
2040         error = ''
2041         try:
2042             vm_dict = {'created': vm_info_dict['created'],
2043                        'description': vm_info_dict['name'],
2044                        'status': vcdStatusCode2manoFormat[int(status_key)],
2045                        'hostId': vm_info_dict['vmuuid'],
2046                        'error_msg': error,
2047                        'vim_info': yaml.safe_dump(vm_info_dict), 'interfaces': []}
2048
2049             if 'interfaces' in vm_info_dict:
2050                 vm_dict['interfaces'] = vm_info_dict['interfaces']
2051             else:
2052                 vm_dict['interfaces'] = []
2053         except KeyError:
2054             vm_dict = {'created': '',
2055                        'description': '',
2056                        'status': vcdStatusCode2manoFormat[int(-1)],
2057                        'hostId': vm_info_dict['vmuuid'],
2058                        'error_msg': "Inconsistency state",
2059                        'vim_info': yaml.safe_dump(vm_info_dict), 'interfaces': []}
2060
2061         return vm_dict
2062
2063     def delete_vminstance(self, vm__vim_uuid, created_items=None):
2064         """Method poweroff and remove VM instance from vcloud director network.
2065
2066         Args:
2067             vm__vim_uuid: VM UUID
2068
2069         Returns:
2070             Returns the instance identifier
2071         """
2072
2073         self.logger.debug("Client requesting delete vm instance {} ".format(vm__vim_uuid))
2074
2075         org, vdc = self.get_vdc_details()
2076         vdc_obj = VDC(self.client, href=vdc.get('href'))
2077         if vdc_obj is None:
2078             self.logger.debug("delete_vminstance(): Failed to get a reference of VDC for a tenant {}".format(
2079                 self.tenant_name))
2080             raise vimconn.vimconnException(
2081                 "delete_vminstance(): Failed to get a reference of VDC for a tenant {}".format(self.tenant_name))
2082
2083         try:
2084             vapp_name = self.get_namebyvappid(vm__vim_uuid)
2085             vapp_resource = vdc_obj.get_vapp(vapp_name)
2086             vapp = VApp(self.client, resource=vapp_resource)
2087             if vapp_name is None:
2088                 self.logger.debug("delete_vminstance(): Failed to get vm by given {} vm uuid".format(vm__vim_uuid))
2089                 return -1, "delete_vminstance(): Failed to get vm by given {} vm uuid".format(vm__vim_uuid)
2090             else:
2091                 self.logger.info("Deleting vApp {} and UUID {}".format(vapp_name, vm__vim_uuid))
2092
2093             # Delete vApp and wait for status change if task executed and vApp is None.
2094
2095             if vapp:
2096                 if vapp_resource.get('deployed') == 'true':
2097                     self.logger.info("Powering off vApp {}".format(vapp_name))
2098                     #Power off vApp
2099                     powered_off = False
2100                     wait_time = 0
2101                     while wait_time <= MAX_WAIT_TIME:
2102                         power_off_task = vapp.power_off()
2103                         result = self.client.get_task_monitor().wait_for_success(task=power_off_task)
2104
2105                         if result.get('status') == 'success':
2106                             powered_off = True
2107                             break
2108                         else:
2109                             self.logger.info("Wait for vApp {} to power off".format(vapp_name))
2110                             time.sleep(INTERVAL_TIME)
2111
2112                         wait_time +=INTERVAL_TIME
2113                     if not powered_off:
2114                         self.logger.debug("delete_vminstance(): Failed to power off VM instance {} ".format(vm__vim_uuid))
2115                     else:
2116                         self.logger.info("delete_vminstance(): Powered off VM instance {} ".format(vm__vim_uuid))
2117
2118                     #Undeploy vApp
2119                     self.logger.info("Undeploy vApp {}".format(vapp_name))
2120                     wait_time = 0
2121                     undeployed = False
2122                     while wait_time <= MAX_WAIT_TIME:
2123                         vapp = VApp(self.client, resource=vapp_resource)
2124                         if not vapp:
2125                             self.logger.debug("delete_vminstance(): Failed to get vm by given {} vm uuid".format(vm__vim_uuid))
2126                             return -1, "delete_vminstance(): Failed to get vm by given {} vm uuid".format(vm__vim_uuid)
2127                         undeploy_task = vapp.undeploy()
2128
2129                         result = self.client.get_task_monitor().wait_for_success(task=undeploy_task)
2130                         if result.get('status') == 'success':
2131                             undeployed = True
2132                             break
2133                         else:
2134                             self.logger.debug("Wait for vApp {} to undeploy".format(vapp_name))
2135                             time.sleep(INTERVAL_TIME)
2136
2137                         wait_time +=INTERVAL_TIME
2138
2139                     if not undeployed:
2140                         self.logger.debug("delete_vminstance(): Failed to undeploy vApp {} ".format(vm__vim_uuid))
2141
2142                 # delete vapp
2143                 self.logger.info("Start deletion of vApp {} ".format(vapp_name))
2144
2145                 if vapp is not None:
2146                     wait_time = 0
2147                     result = False
2148
2149                     while wait_time <= MAX_WAIT_TIME:
2150                         vapp = VApp(self.client, resource=vapp_resource)
2151                         if not vapp:
2152                             self.logger.debug("delete_vminstance(): Failed to get vm by given {} vm uuid".format(vm__vim_uuid))
2153                             return -1, "delete_vminstance(): Failed to get vm by given {} vm uuid".format(vm__vim_uuid)
2154
2155                         delete_task = vdc_obj.delete_vapp(vapp.name, force=True)
2156
2157                         result = self.client.get_task_monitor().wait_for_success(task=delete_task)
2158                         if result.get('status') == 'success':
2159                             break
2160                         else:
2161                             self.logger.debug("Wait for vApp {} to delete".format(vapp_name))
2162                             time.sleep(INTERVAL_TIME)
2163
2164                         wait_time +=INTERVAL_TIME
2165
2166                     if result is None:
2167                         self.logger.debug("delete_vminstance(): Failed delete uuid {} ".format(vm__vim_uuid))
2168                     else:
2169                         self.logger.info("Deleted vm instance {} sccessfully".format(vm__vim_uuid))
2170                         return vm__vim_uuid
2171         except:
2172             self.logger.debug(traceback.format_exc())
2173             raise vimconn.vimconnException("delete_vminstance(): Failed delete vm instance {}".format(vm__vim_uuid))
2174
2175
2176     def refresh_vms_status(self, vm_list):
2177         """Get the status of the virtual machines and their interfaces/ports
2178            Params: the list of VM identifiers
2179            Returns a dictionary with:
2180                 vm_id:          #VIM id of this Virtual Machine
2181                     status:     #Mandatory. Text with one of:
2182                                 #  DELETED (not found at vim)
2183                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
2184                                 #  OTHER (Vim reported other status not understood)
2185                                 #  ERROR (VIM indicates an ERROR status)
2186                                 #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
2187                                 #  CREATING (on building process), ERROR
2188                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
2189                                 #
2190                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
2191                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
2192                     interfaces:
2193                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
2194                         mac_address:      #Text format XX:XX:XX:XX:XX:XX
2195                         vim_net_id:       #network id where this interface is connected
2196                         vim_interface_id: #interface/port VIM id
2197                         ip_address:       #null, or text with IPv4, IPv6 address
2198         """
2199
2200         self.logger.debug("Client requesting refresh vm status for {} ".format(vm_list))
2201
2202         org,vdc = self.get_vdc_details()
2203         if vdc is None:
2204             raise vimconn.vimconnException("Failed to get a reference of VDC for a tenant {}".format(self.tenant_name))
2205
2206         vms_dict = {}
2207         nsx_edge_list = []
2208         for vmuuid in vm_list:
2209             vapp_name = self.get_namebyvappid(vmuuid)
2210             if vapp_name is not None:
2211
2212                 try:
2213                     vm_pci_details = self.get_vm_pci_details(vmuuid)
2214                     vdc_obj = VDC(self.client, href=vdc.get('href'))
2215                     vapp_resource = vdc_obj.get_vapp(vapp_name)
2216                     the_vapp = VApp(self.client, resource=vapp_resource)
2217
2218                     vm_details = {}
2219                     for vm in the_vapp.get_all_vms():
2220                         headers = {'Accept':'application/*+xml;version=' + API_VERSION,
2221                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
2222                         response = self.perform_request(req_type='GET',
2223                                                         url=vm.get('href'),
2224                                                         headers=headers)
2225
2226                         if response.status_code != 200:
2227                             self.logger.error("refresh_vms_status : REST call {} failed reason : {}"\
2228                                                             "status code : {}".format(vm.get('href'),
2229                                                                                     response.content,
2230                                                                                response.status_code))
2231                             raise vimconn.vimconnException("refresh_vms_status : Failed to get "\
2232                                                                          "VM details")
2233                         xmlroot = XmlElementTree.fromstring(response.content)
2234
2235                         result = response.content.replace("\n"," ")
2236                         hdd_mb = re.search('vcloud:capacity="(\d+)"\svcloud:storageProfileOverrideVmDefault=',result).group(1)
2237                         vm_details['hdd_mb'] = int(hdd_mb) if hdd_mb else None
2238                         cpus = re.search('<rasd:Description>Number of Virtual CPUs</.*?>(\d+)</rasd:VirtualQuantity>',result).group(1)
2239                         vm_details['cpus'] = int(cpus) if cpus else None
2240                         memory_mb = re.search('<rasd:Description>Memory Size</.*?>(\d+)</rasd:VirtualQuantity>',result).group(1)
2241                         vm_details['memory_mb'] = int(memory_mb) if memory_mb else None
2242                         vm_details['status'] = vcdStatusCode2manoFormat[int(xmlroot.get('status'))]
2243                         vm_details['id'] = xmlroot.get('id')
2244                         vm_details['name'] = xmlroot.get('name')
2245                         vm_info = [vm_details]
2246                         if vm_pci_details:
2247                             vm_info[0].update(vm_pci_details)
2248
2249                         vm_dict = {'status': vcdStatusCode2manoFormat[int(vapp_resource.get('status'))],
2250                                    'error_msg': vcdStatusCode2manoFormat[int(vapp_resource.get('status'))],
2251                                    'vim_info': yaml.safe_dump(vm_info), 'interfaces': []}
2252
2253                         # get networks
2254                         vm_ip = None
2255                         vm_mac = None
2256                         if vm.NetworkConnectionSection.NetworkConnection:
2257                             vm_mac = vm.NetworkConnectionSection.NetworkConnection.MACAddress
2258                         if vm_ip is None:
2259                             if not nsx_edge_list:
2260                                 nsx_edge_list = self.get_edge_details()
2261                                 if nsx_edge_list is None:
2262                                     raise vimconn.vimconnException("refresh_vms_status:"\
2263                                                                       "Failed to get edge details from NSX Manager")
2264                             if vm_mac is not None:
2265                                 vm_ip = self.get_ipaddr_from_NSXedge(nsx_edge_list, vm_mac)
2266
2267                         network_name = vm.NetworkConnectionSection.NetworkConnection.get('network')
2268                         vm_net_id = self.get_network_id_by_name(network_name)
2269                         interface = {"mac_address": vm_mac,
2270                                      "vim_net_id": vm_net_id,
2271                                      "vim_interface_id": vm_net_id,
2272                                      'ip_address': vm_ip}
2273
2274                         vm_dict["interfaces"].append(interface)
2275
2276                     # add a vm to vm dict
2277                     vms_dict.setdefault(vmuuid, vm_dict)
2278                     self.logger.debug("refresh_vms_status : vm info {}".format(vm_dict))
2279                 except Exception as exp:
2280                     self.logger.debug("Error in response {}".format(exp))
2281                     self.logger.debug(traceback.format_exc())
2282
2283         return vms_dict
2284
2285
2286     def get_edge_details(self):
2287         """Get the NSX edge list from NSX Manager
2288            Returns list of NSX edges
2289         """
2290         edge_list = []
2291         rheaders = {'Content-Type': 'application/xml'}
2292         nsx_api_url = '/api/4.0/edges'
2293
2294         self.logger.debug("Get edge details from NSX Manager {} {}".format(self.nsx_manager, nsx_api_url))
2295
2296         try:
2297             resp = requests.get(self.nsx_manager + nsx_api_url,
2298                                 auth = (self.nsx_user, self.nsx_password),
2299                                 verify = False, headers = rheaders)
2300             if resp.status_code == requests.codes.ok:
2301                 paged_Edge_List = XmlElementTree.fromstring(resp.text)
2302                 for edge_pages in paged_Edge_List:
2303                     if edge_pages.tag == 'edgePage':
2304                         for edge_summary in edge_pages:
2305                             if edge_summary.tag == 'pagingInfo':
2306                                 for element in edge_summary:
2307                                     if element.tag == 'totalCount' and element.text == '0':
2308                                         raise vimconn.vimconnException("get_edge_details: No NSX edges details found: {}"
2309                                                                        .format(self.nsx_manager))
2310
2311                             if edge_summary.tag == 'edgeSummary':
2312                                 for element in edge_summary:
2313                                     if element.tag == 'id':
2314                                         edge_list.append(element.text)
2315                     else:
2316                         raise vimconn.vimconnException("get_edge_details: No NSX edge details found: {}"
2317                                                        .format(self.nsx_manager))
2318
2319                 if not edge_list:
2320                     raise vimconn.vimconnException("get_edge_details: "\
2321                                                    "No NSX edge details found: {}"
2322                                                    .format(self.nsx_manager))
2323                 else:
2324                     self.logger.debug("get_edge_details: Found NSX edges {}".format(edge_list))
2325                     return edge_list
2326             else:
2327                 self.logger.debug("get_edge_details: "
2328                                   "Failed to get NSX edge details from NSX Manager: {}"
2329                                   .format(resp.content))
2330                 return None
2331
2332         except Exception as exp:
2333             self.logger.debug("get_edge_details: "\
2334                               "Failed to get NSX edge details from NSX Manager: {}"
2335                               .format(exp))
2336             raise vimconn.vimconnException("get_edge_details: "\
2337                                            "Failed to get NSX edge details from NSX Manager: {}"
2338                                            .format(exp))
2339
2340
2341     def get_ipaddr_from_NSXedge(self, nsx_edges, mac_address):
2342         """Get IP address details from NSX edges, using the MAC address
2343            PARAMS: nsx_edges : List of NSX edges
2344                    mac_address : Find IP address corresponding to this MAC address
2345            Returns: IP address corrresponding to the provided MAC address
2346         """
2347
2348         ip_addr = None
2349         rheaders = {'Content-Type': 'application/xml'}
2350
2351         self.logger.debug("get_ipaddr_from_NSXedge: Finding IP addr from NSX edge")
2352
2353         try:
2354             for edge in nsx_edges:
2355                 nsx_api_url = '/api/4.0/edges/'+ edge +'/dhcp/leaseInfo'
2356
2357                 resp = requests.get(self.nsx_manager + nsx_api_url,
2358                                     auth = (self.nsx_user, self.nsx_password),
2359                                     verify = False, headers = rheaders)
2360
2361                 if resp.status_code == requests.codes.ok:
2362                     dhcp_leases = XmlElementTree.fromstring(resp.text)
2363                     for child in dhcp_leases:
2364                         if child.tag == 'dhcpLeaseInfo':
2365                             dhcpLeaseInfo = child
2366                             for leaseInfo in dhcpLeaseInfo:
2367                                 for elem in leaseInfo:
2368                                     if (elem.tag)=='macAddress':
2369                                         edge_mac_addr = elem.text
2370                                     if (elem.tag)=='ipAddress':
2371                                         ip_addr = elem.text
2372                                 if edge_mac_addr is not None:
2373                                     if edge_mac_addr == mac_address:
2374                                         self.logger.debug("Found ip addr {} for mac {} at NSX edge {}"
2375                                                           .format(ip_addr, mac_address,edge))
2376                                         return ip_addr
2377                 else:
2378                     self.logger.debug("get_ipaddr_from_NSXedge: "\
2379                                       "Error occurred while getting DHCP lease info from NSX Manager: {}"
2380                                       .format(resp.content))
2381
2382             self.logger.debug("get_ipaddr_from_NSXedge: No IP addr found in any NSX edge")
2383             return None
2384
2385         except XmlElementTree.ParseError as Err:
2386             self.logger.debug("ParseError in response from NSX Manager {}".format(Err.message), exc_info=True)
2387
2388
2389     def action_vminstance(self, vm__vim_uuid=None, action_dict=None, created_items={}):
2390         """Send and action over a VM instance from VIM
2391         Returns the vm_id if the action was successfully sent to the VIM"""
2392
2393         self.logger.debug("Received action for vm {} and action dict {}".format(vm__vim_uuid, action_dict))
2394         if vm__vim_uuid is None or action_dict is None:
2395             raise vimconn.vimconnException("Invalid request. VM id or action is None.")
2396
2397         org, vdc = self.get_vdc_details()
2398         if vdc is None:
2399             raise  vimconn.vimconnException("Failed to get a reference of VDC for a tenant {}".format(self.tenant_name))
2400
2401         vapp_name = self.get_namebyvappid(vm__vim_uuid)
2402         if vapp_name is None:
2403             self.logger.debug("action_vminstance(): Failed to get vm by given {} vm uuid".format(vm__vim_uuid))
2404             raise vimconn.vimconnException("Failed to get vm by given {} vm uuid".format(vm__vim_uuid))
2405         else:
2406             self.logger.info("Action_vminstance vApp {} and UUID {}".format(vapp_name, vm__vim_uuid))
2407
2408         try:
2409             vdc_obj = VDC(self.client, href=vdc.get('href'))
2410             vapp_resource = vdc_obj.get_vapp(vapp_name)
2411             vapp = VApp(self.client, resource=vapp_resource)
2412             if "start" in action_dict:
2413                 self.logger.info("action_vminstance: Power on vApp: {}".format(vapp_name))
2414                 poweron_task = self.power_on_vapp(vm__vim_uuid, vapp_name)
2415                 result = self.client.get_task_monitor().wait_for_success(task=poweron_task)
2416                 self.instance_actions_result("start", result, vapp_name)
2417             elif "rebuild" in action_dict:
2418                 self.logger.info("action_vminstance: Rebuild vApp: {}".format(vapp_name))
2419                 rebuild_task = vapp.deploy(power_on=True)
2420                 result = self.client.get_task_monitor().wait_for_success(task=rebuild_task)
2421                 self.instance_actions_result("rebuild", result, vapp_name)
2422             elif "pause" in action_dict:
2423                 self.logger.info("action_vminstance: pause vApp: {}".format(vapp_name))
2424                 pause_task = vapp.undeploy(action='suspend')
2425                 result = self.client.get_task_monitor().wait_for_success(task=pause_task)
2426                 self.instance_actions_result("pause", result, vapp_name)
2427             elif "resume" in action_dict:
2428                 self.logger.info("action_vminstance: resume vApp: {}".format(vapp_name))
2429                 poweron_task = self.power_on_vapp(vm__vim_uuid, vapp_name)
2430                 result = self.client.get_task_monitor().wait_for_success(task=poweron_task)
2431                 self.instance_actions_result("resume", result, vapp_name)
2432             elif "shutoff" in action_dict or "shutdown" in action_dict:
2433                 action_name , value = action_dict.items()[0]
2434                 #For python3
2435                 #action_name , value = list(action_dict.items())[0]
2436                 self.logger.info("action_vminstance: {} vApp: {}".format(action_name, vapp_name))
2437                 shutdown_task = vapp.shutdown()
2438                 result = self.client.get_task_monitor().wait_for_success(task=shutdown_task)
2439                 if action_name == "shutdown":
2440                     self.instance_actions_result("shutdown", result, vapp_name)
2441                 else:
2442                     self.instance_actions_result("shutoff", result, vapp_name)
2443             elif "forceOff" in action_dict:
2444                 result = vapp.undeploy(action='powerOff')
2445                 self.instance_actions_result("forceOff", result, vapp_name)
2446             elif "reboot" in action_dict:
2447                 self.logger.info("action_vminstance: reboot vApp: {}".format(vapp_name))
2448                 reboot_task = vapp.reboot()
2449                 self.client.get_task_monitor().wait_for_success(task=reboot_task)
2450             else:
2451                 raise vimconn.vimconnException("action_vminstance: Invalid action {} or action is None.".format(action_dict))
2452             return vm__vim_uuid
2453         except Exception as exp :
2454             self.logger.debug("action_vminstance: Failed with Exception {}".format(exp))
2455             raise vimconn.vimconnException("action_vminstance: Failed with Exception {}".format(exp))
2456
2457     def instance_actions_result(self, action, result, vapp_name):
2458         if result.get('status') == 'success':
2459             self.logger.info("action_vminstance: Sucessfully {} the vApp: {}".format(action, vapp_name))
2460         else:
2461             self.logger.error("action_vminstance: Failed to {} vApp: {}".format(action, vapp_name))
2462
2463     def get_vminstance_console(self, vm_id, console_type="vnc"):
2464         """
2465         Get a console for the virtual machine
2466         Params:
2467             vm_id: uuid of the VM
2468             console_type, can be:
2469                 "novnc" (by default), "xvpvnc" for VNC types,
2470                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
2471         Returns dict with the console parameters:
2472                 protocol: ssh, ftp, http, https, ...
2473                 server:   usually ip address
2474                 port:     the http, ssh, ... port
2475                 suffix:   extra text, e.g. the http path and query string
2476         """
2477         raise vimconn.vimconnNotImplemented("Should have implemented this")
2478
2479     # NOT USED METHODS in current version
2480
2481     def host_vim2gui(self, host, server_dict):
2482         """Transform host dictionary from VIM format to GUI format,
2483         and append to the server_dict
2484         """
2485         raise vimconn.vimconnNotImplemented("Should have implemented this")
2486
2487     def get_hosts_info(self):
2488         """Get the information of deployed hosts
2489         Returns the hosts content"""
2490         raise vimconn.vimconnNotImplemented("Should have implemented this")
2491
2492     def get_hosts(self, vim_tenant):
2493         """Get the hosts and deployed instances
2494         Returns the hosts content"""
2495         raise vimconn.vimconnNotImplemented("Should have implemented this")
2496
2497     def get_processor_rankings(self):
2498         """Get the processor rankings in the VIM database"""
2499         raise vimconn.vimconnNotImplemented("Should have implemented this")
2500
2501     def new_host(self, host_data):
2502         """Adds a new host to VIM"""
2503         '''Returns status code of the VIM response'''
2504         raise vimconn.vimconnNotImplemented("Should have implemented this")
2505
2506     def new_external_port(self, port_data):
2507         """Adds a external port to VIM"""
2508         '''Returns the port identifier'''
2509         raise vimconn.vimconnNotImplemented("Should have implemented this")
2510
2511     def new_external_network(self, net_name, net_type):
2512         """Adds a external network to VIM (shared)"""
2513         '''Returns the network identifier'''
2514         raise vimconn.vimconnNotImplemented("Should have implemented this")
2515
2516     def connect_port_network(self, port_id, network_id, admin=False):
2517         """Connects a external port to a network"""
2518         '''Returns status code of the VIM response'''
2519         raise vimconn.vimconnNotImplemented("Should have implemented this")
2520
2521     def new_vminstancefromJSON(self, vm_data):
2522         """Adds a VM instance to VIM"""
2523         '''Returns the instance identifier'''
2524         raise vimconn.vimconnNotImplemented("Should have implemented this")
2525
2526     def get_network_name_by_id(self, network_uuid=None):
2527         """Method gets vcloud director network named based on supplied uuid.
2528
2529         Args:
2530             network_uuid: network_id
2531
2532         Returns:
2533             The return network name.
2534         """
2535
2536         if not network_uuid:
2537             return None
2538
2539         try:
2540             org_dict = self.get_org(self.org_uuid)
2541             if 'networks' in org_dict:
2542                 org_network_dict = org_dict['networks']
2543                 for net_uuid in org_network_dict:
2544                     if net_uuid == network_uuid:
2545                         return org_network_dict[net_uuid]
2546         except:
2547             self.logger.debug("Exception in get_network_name_by_id")
2548             self.logger.debug(traceback.format_exc())
2549
2550         return None
2551
2552     def get_network_id_by_name(self, network_name=None):
2553         """Method gets vcloud director network uuid based on supplied name.
2554
2555         Args:
2556             network_name: network_name
2557         Returns:
2558             The return network uuid.
2559             network_uuid: network_id
2560         """
2561
2562         if not network_name:
2563             self.logger.debug("get_network_id_by_name() : Network name is empty")
2564             return None
2565
2566         try:
2567             org_dict = self.get_org(self.org_uuid)
2568             if org_dict and 'networks' in org_dict:
2569                 org_network_dict = org_dict['networks']
2570                 for net_uuid,net_name in org_network_dict.iteritems():
2571                 #For python3
2572                 #for net_uuid,net_name in org_network_dict.items():
2573                     if net_name == network_name:
2574                         return net_uuid
2575
2576         except KeyError as exp:
2577             self.logger.debug("get_network_id_by_name() : KeyError- {} ".format(exp))
2578
2579         return None
2580
2581     def list_org_action(self):
2582         """
2583         Method leverages vCloud director and query for available organization for particular user
2584
2585         Args:
2586             vca - is active VCA connection.
2587             vdc_name - is a vdc name that will be used to query vms action
2588
2589             Returns:
2590                 The return XML respond
2591         """
2592         url_list = [self.url, '/api/org']
2593         vm_list_rest_call = ''.join(url_list)
2594
2595         if self.client._session:
2596             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
2597                        'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
2598
2599             response = self.perform_request(req_type='GET',
2600                                      url=vm_list_rest_call,
2601                                            headers=headers)
2602
2603             if response.status_code == 403:
2604                 response = self.retry_rest('GET', vm_list_rest_call)
2605
2606             if response.status_code == requests.codes.ok:
2607                 return response.content
2608
2609         return None
2610
2611     def get_org_action(self, org_uuid=None):
2612         """
2613         Method leverages vCloud director and retrieve available object for organization.
2614
2615         Args:
2616             org_uuid - vCD organization uuid
2617             self.client - is active connection.
2618
2619             Returns:
2620                 The return XML respond
2621         """
2622
2623         if org_uuid is None:
2624             return None
2625
2626         url_list = [self.url, '/api/org/', org_uuid]
2627         vm_list_rest_call = ''.join(url_list)
2628
2629         if self.client._session:
2630             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
2631                      'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
2632
2633             #response = requests.get(vm_list_rest_call, headers=headers, verify=False)
2634             response = self.perform_request(req_type='GET',
2635                                             url=vm_list_rest_call,
2636                                             headers=headers)
2637             if response.status_code == 403:
2638                 response = self.retry_rest('GET', vm_list_rest_call)
2639
2640             if response.status_code == requests.codes.ok:
2641                 return response.content
2642         return None
2643
2644     def get_org(self, org_uuid=None):
2645         """
2646         Method retrieves available organization in vCloud Director
2647
2648         Args:
2649             org_uuid - is a organization uuid.
2650
2651             Returns:
2652                 The return dictionary with following key
2653                     "network" - for network list under the org
2654                     "catalogs" - for network list under the org
2655                     "vdcs" - for vdc list under org
2656         """
2657
2658         org_dict = {}
2659
2660         if org_uuid is None:
2661             return org_dict
2662
2663         content = self.get_org_action(org_uuid=org_uuid)
2664         try:
2665             vdc_list = {}
2666             network_list = {}
2667             catalog_list = {}
2668             vm_list_xmlroot = XmlElementTree.fromstring(content)
2669             for child in vm_list_xmlroot:
2670                 if child.attrib['type'] == 'application/vnd.vmware.vcloud.vdc+xml':
2671                     vdc_list[child.attrib['href'].split("/")[-1:][0]] = child.attrib['name']
2672                     org_dict['vdcs'] = vdc_list
2673                 if child.attrib['type'] == 'application/vnd.vmware.vcloud.orgNetwork+xml':
2674                     network_list[child.attrib['href'].split("/")[-1:][0]] = child.attrib['name']
2675                     org_dict['networks'] = network_list
2676                 if child.attrib['type'] == 'application/vnd.vmware.vcloud.catalog+xml':
2677                     catalog_list[child.attrib['href'].split("/")[-1:][0]] = child.attrib['name']
2678                     org_dict['catalogs'] = catalog_list
2679         except:
2680             pass
2681
2682         return org_dict
2683
2684     def get_org_list(self):
2685         """
2686         Method retrieves available organization in vCloud Director
2687
2688         Args:
2689             vca - is active VCA connection.
2690
2691             Returns:
2692                 The return dictionary and key for each entry VDC UUID
2693         """
2694
2695         org_dict = {}
2696
2697         content = self.list_org_action()
2698         try:
2699             vm_list_xmlroot = XmlElementTree.fromstring(content)
2700             for vm_xml in vm_list_xmlroot:
2701                 if vm_xml.tag.split("}")[1] == 'Org':
2702                     org_uuid = vm_xml.attrib['href'].split('/')[-1:]
2703                     org_dict[org_uuid[0]] = vm_xml.attrib['name']
2704         except:
2705             pass
2706
2707         return org_dict
2708
2709     def vms_view_action(self, vdc_name=None):
2710         """ Method leverages vCloud director vms query call
2711
2712         Args:
2713             vca - is active VCA connection.
2714             vdc_name - is a vdc name that will be used to query vms action
2715
2716             Returns:
2717                 The return XML respond
2718         """
2719         vca = self.connect()
2720         if vdc_name is None:
2721             return None
2722
2723         url_list = [vca.host, '/api/vms/query']
2724         vm_list_rest_call = ''.join(url_list)
2725
2726         if not (not vca.vcloud_session or not vca.vcloud_session.organization):
2727             refs = filter(lambda ref: ref.name == vdc_name and ref.type_ == 'application/vnd.vmware.vcloud.vdc+xml',
2728                           vca.vcloud_session.organization.Link)
2729             #For python3
2730             #refs = [ref for ref in vca.vcloud_session.organization.Link if ref.name == vdc_name and\
2731             #        ref.type_ == 'application/vnd.vmware.vcloud.vdc+xml']
2732             if len(refs) == 1:
2733                 response = Http.get(url=vm_list_rest_call,
2734                                     headers=vca.vcloud_session.get_vcloud_headers(),
2735                                     verify=vca.verify,
2736                                     logger=vca.logger)
2737                 if response.status_code == requests.codes.ok:
2738                     return response.content
2739
2740         return None
2741
2742     def get_vapp_list(self, vdc_name=None):
2743         """
2744         Method retrieves vApp list deployed vCloud director and returns a dictionary
2745         contains a list of all vapp deployed for queried VDC.
2746         The key for a dictionary is vApp UUID
2747
2748
2749         Args:
2750             vca - is active VCA connection.
2751             vdc_name - is a vdc name that will be used to query vms action
2752
2753             Returns:
2754                 The return dictionary and key for each entry vapp UUID
2755         """
2756
2757         vapp_dict = {}
2758         if vdc_name is None:
2759             return vapp_dict
2760
2761         content = self.vms_view_action(vdc_name=vdc_name)
2762         try:
2763             vm_list_xmlroot = XmlElementTree.fromstring(content)
2764             for vm_xml in vm_list_xmlroot:
2765                 if vm_xml.tag.split("}")[1] == 'VMRecord':
2766                     if vm_xml.attrib['isVAppTemplate'] == 'true':
2767                         rawuuid = vm_xml.attrib['container'].split('/')[-1:]
2768                         if 'vappTemplate-' in rawuuid[0]:
2769                             # vm in format vappTemplate-e63d40e7-4ff5-4c6d-851f-96c1e4da86a5 we remove
2770                             # vm and use raw UUID as key
2771                             vapp_dict[rawuuid[0][13:]] = vm_xml.attrib
2772         except:
2773             pass
2774
2775         return vapp_dict
2776
2777     def get_vm_list(self, vdc_name=None):
2778         """
2779         Method retrieves VM's list deployed vCloud director. It returns a dictionary
2780         contains a list of all VM's deployed for queried VDC.
2781         The key for a dictionary is VM UUID
2782
2783
2784         Args:
2785             vca - is active VCA connection.
2786             vdc_name - is a vdc name that will be used to query vms action
2787
2788             Returns:
2789                 The return dictionary and key for each entry vapp UUID
2790         """
2791         vm_dict = {}
2792
2793         if vdc_name is None:
2794             return vm_dict
2795
2796         content = self.vms_view_action(vdc_name=vdc_name)
2797         try:
2798             vm_list_xmlroot = XmlElementTree.fromstring(content)
2799             for vm_xml in vm_list_xmlroot:
2800                 if vm_xml.tag.split("}")[1] == 'VMRecord':
2801                     if vm_xml.attrib['isVAppTemplate'] == 'false':
2802                         rawuuid = vm_xml.attrib['href'].split('/')[-1:]
2803                         if 'vm-' in rawuuid[0]:
2804                             # vm in format vm-e63d40e7-4ff5-4c6d-851f-96c1e4da86a5 we remove
2805                             #  vm and use raw UUID as key
2806                             vm_dict[rawuuid[0][3:]] = vm_xml.attrib
2807         except:
2808             pass
2809
2810         return vm_dict
2811
2812     def get_vapp(self, vdc_name=None, vapp_name=None, isuuid=False):
2813         """
2814         Method retrieves VM deployed vCloud director. It returns VM attribute as dictionary
2815         contains a list of all VM's deployed for queried VDC.
2816         The key for a dictionary is VM UUID
2817
2818
2819         Args:
2820             vca - is active VCA connection.
2821             vdc_name - is a vdc name that will be used to query vms action
2822
2823             Returns:
2824                 The return dictionary and key for each entry vapp UUID
2825         """
2826         vm_dict = {}
2827         vca = self.connect()
2828         if not vca:
2829             raise vimconn.vimconnConnectionException("self.connect() is failed")
2830
2831         if vdc_name is None:
2832             return vm_dict
2833
2834         content = self.vms_view_action(vdc_name=vdc_name)
2835         try:
2836             vm_list_xmlroot = XmlElementTree.fromstring(content)
2837             for vm_xml in vm_list_xmlroot:
2838                 if vm_xml.tag.split("}")[1] == 'VMRecord' and vm_xml.attrib['isVAppTemplate'] == 'false':
2839                     # lookup done by UUID
2840                     if isuuid:
2841                         if vapp_name in vm_xml.attrib['container']:
2842                             rawuuid = vm_xml.attrib['href'].split('/')[-1:]
2843                             if 'vm-' in rawuuid[0]:
2844                                 vm_dict[rawuuid[0][3:]] = vm_xml.attrib
2845                                 break
2846                     # lookup done by Name
2847                     else:
2848                         if vapp_name in vm_xml.attrib['name']:
2849                             rawuuid = vm_xml.attrib['href'].split('/')[-1:]
2850                             if 'vm-' in rawuuid[0]:
2851                                 vm_dict[rawuuid[0][3:]] = vm_xml.attrib
2852                                 break
2853         except:
2854             pass
2855
2856         return vm_dict
2857
2858     def get_network_action(self, network_uuid=None):
2859         """
2860         Method leverages vCloud director and query network based on network uuid
2861
2862         Args:
2863             vca - is active VCA connection.
2864             network_uuid - is a network uuid
2865
2866             Returns:
2867                 The return XML respond
2868         """
2869
2870         if network_uuid is None:
2871             return None
2872
2873         url_list = [self.url, '/api/network/', network_uuid]
2874         vm_list_rest_call = ''.join(url_list)
2875
2876         if self.client._session:
2877             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
2878                      'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
2879
2880             response = self.perform_request(req_type='GET',
2881                                             url=vm_list_rest_call,
2882                                             headers=headers)
2883             #Retry login if session expired & retry sending request
2884             if response.status_code == 403:
2885                 response = self.retry_rest('GET', vm_list_rest_call)
2886
2887             if response.status_code == requests.codes.ok:
2888                 return response.content
2889
2890         return None
2891
2892     def get_vcd_network(self, network_uuid=None):
2893         """
2894         Method retrieves available network from vCloud Director
2895
2896         Args:
2897             network_uuid - is VCD network UUID
2898
2899         Each element serialized as key : value pair
2900
2901         Following keys available for access.    network_configuration['Gateway'}
2902         <Configuration>
2903           <IpScopes>
2904             <IpScope>
2905                 <IsInherited>true</IsInherited>
2906                 <Gateway>172.16.252.100</Gateway>
2907                 <Netmask>255.255.255.0</Netmask>
2908                 <Dns1>172.16.254.201</Dns1>
2909                 <Dns2>172.16.254.202</Dns2>
2910                 <DnsSuffix>vmwarelab.edu</DnsSuffix>
2911                 <IsEnabled>true</IsEnabled>
2912                 <IpRanges>
2913                     <IpRange>
2914                         <StartAddress>172.16.252.1</StartAddress>
2915                         <EndAddress>172.16.252.99</EndAddress>
2916                     </IpRange>
2917                 </IpRanges>
2918             </IpScope>
2919         </IpScopes>
2920         <FenceMode>bridged</FenceMode>
2921
2922         Returns:
2923                 The return dictionary and key for each entry vapp UUID
2924         """
2925
2926         network_configuration = {}
2927         if network_uuid is None:
2928             return network_uuid
2929
2930         try:
2931             content = self.get_network_action(network_uuid=network_uuid)
2932             vm_list_xmlroot = XmlElementTree.fromstring(content)
2933
2934             network_configuration['status'] = vm_list_xmlroot.get("status")
2935             network_configuration['name'] = vm_list_xmlroot.get("name")
2936             network_configuration['uuid'] = vm_list_xmlroot.get("id").split(":")[3]
2937
2938             for child in vm_list_xmlroot:
2939                 if child.tag.split("}")[1] == 'IsShared':
2940                     network_configuration['isShared'] = child.text.strip()
2941                 if child.tag.split("}")[1] == 'Configuration':
2942                     for configuration in child.iter():
2943                         tagKey = configuration.tag.split("}")[1].strip()
2944                         if tagKey != "":
2945                             network_configuration[tagKey] = configuration.text.strip()
2946             return network_configuration
2947         except Exception as exp :
2948             self.logger.debug("get_vcd_network: Failed with Exception {}".format(exp))
2949             raise vimconn.vimconnException("get_vcd_network: Failed with Exception {}".format(exp))
2950
2951         return network_configuration
2952
2953     def delete_network_action(self, network_uuid=None):
2954         """
2955         Method delete given network from vCloud director
2956
2957         Args:
2958             network_uuid - is a network uuid that client wish to delete
2959
2960             Returns:
2961                 The return None or XML respond or false
2962         """
2963         client = self.connect_as_admin()
2964         if not client:
2965             raise vimconn.vimconnConnectionException("Failed to connect vCD as admin")
2966         if network_uuid is None:
2967             return False
2968
2969         url_list = [self.url, '/api/admin/network/', network_uuid]
2970         vm_list_rest_call = ''.join(url_list)
2971
2972         if client._session:
2973             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
2974                      'x-vcloud-authorization': client._session.headers['x-vcloud-authorization']}
2975             response = self.perform_request(req_type='DELETE',
2976                                             url=vm_list_rest_call,
2977                                             headers=headers)
2978             if response.status_code == 202:
2979                 return True
2980
2981         return False
2982
2983     def create_network(self, network_name=None, net_type='bridge', parent_network_uuid=None,
2984                        ip_profile=None, isshared='true'):
2985         """
2986         Method create network in vCloud director
2987
2988         Args:
2989             network_name - is network name to be created.
2990             net_type - can be 'bridge','data','ptp','mgmt'.
2991             ip_profile is a dict containing the IP parameters of the network
2992             isshared - is a boolean
2993             parent_network_uuid - is parent provider vdc network that will be used for mapping.
2994             It optional attribute. by default if no parent network indicate the first available will be used.
2995
2996             Returns:
2997                 The return network uuid or return None
2998         """
2999
3000         new_network_name = [network_name, '-', str(uuid.uuid4())]
3001         content = self.create_network_rest(network_name=''.join(new_network_name),
3002                                            ip_profile=ip_profile,
3003                                            net_type=net_type,
3004                                            parent_network_uuid=parent_network_uuid,
3005                                            isshared=isshared)
3006         if content is None:
3007             self.logger.debug("Failed create network {}.".format(network_name))
3008             return None
3009
3010         try:
3011             vm_list_xmlroot = XmlElementTree.fromstring(content)
3012             vcd_uuid = vm_list_xmlroot.get('id').split(":")
3013             if len(vcd_uuid) == 4:
3014                 self.logger.info("Created new network name: {} uuid: {}".format(network_name, vcd_uuid[3]))
3015                 return vcd_uuid[3]
3016         except:
3017             self.logger.debug("Failed create network {}".format(network_name))
3018             return None
3019
3020     def create_network_rest(self, network_name=None, net_type='bridge', parent_network_uuid=None,
3021                             ip_profile=None, isshared='true'):
3022         """
3023         Method create network in vCloud director
3024
3025         Args:
3026             network_name - is network name to be created.
3027             net_type - can be 'bridge','data','ptp','mgmt'.
3028             ip_profile is a dict containing the IP parameters of the network
3029             isshared - is a boolean
3030             parent_network_uuid - is parent provider vdc network that will be used for mapping.
3031             It optional attribute. by default if no parent network indicate the first available will be used.
3032
3033             Returns:
3034                 The return network uuid or return None
3035         """
3036         client_as_admin = self.connect_as_admin()
3037         if not client_as_admin:
3038             raise vimconn.vimconnConnectionException("Failed to connect vCD.")
3039         if network_name is None:
3040             return None
3041
3042         url_list = [self.url, '/api/admin/vdc/', self.tenant_id]
3043         vm_list_rest_call = ''.join(url_list)
3044
3045         if client_as_admin._session:
3046             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
3047                      'x-vcloud-authorization': client_as_admin._session.headers['x-vcloud-authorization']}
3048
3049             response = self.perform_request(req_type='GET',
3050                                             url=vm_list_rest_call,
3051                                             headers=headers)
3052
3053             provider_network = None
3054             available_networks = None
3055             add_vdc_rest_url = None
3056
3057             if response.status_code != requests.codes.ok:
3058                 self.logger.debug("REST API call {} failed. Return status code {}".format(vm_list_rest_call,
3059                                                                                           response.status_code))
3060                 return None
3061             else:
3062                 try:
3063                     vm_list_xmlroot = XmlElementTree.fromstring(response.content)
3064                     for child in vm_list_xmlroot:
3065                         if child.tag.split("}")[1] == 'ProviderVdcReference':
3066                             provider_network = child.attrib.get('href')
3067                             # application/vnd.vmware.admin.providervdc+xml
3068                         if child.tag.split("}")[1] == 'Link':
3069                             if child.attrib.get('type') == 'application/vnd.vmware.vcloud.orgVdcNetwork+xml' \
3070                                     and child.attrib.get('rel') == 'add':
3071                                 add_vdc_rest_url = child.attrib.get('href')
3072                 except:
3073                     self.logger.debug("Failed parse respond for rest api call {}".format(vm_list_rest_call))
3074                     self.logger.debug("Respond body {}".format(response.content))
3075                     return None
3076
3077             # find  pvdc provided available network
3078             response = self.perform_request(req_type='GET',
3079                                             url=provider_network,
3080                                             headers=headers)
3081             if response.status_code != requests.codes.ok:
3082                 self.logger.debug("REST API call {} failed. Return status code {}".format(vm_list_rest_call,
3083                                                                                           response.status_code))
3084                 return None
3085
3086             if parent_network_uuid is None:
3087                 try:
3088                     vm_list_xmlroot = XmlElementTree.fromstring(response.content)
3089                     for child in vm_list_xmlroot.iter():
3090                         if child.tag.split("}")[1] == 'AvailableNetworks':
3091                             for networks in child.iter():
3092                                 # application/vnd.vmware.admin.network+xml
3093                                 if networks.attrib.get('href') is not None:
3094                                     available_networks = networks.attrib.get('href')
3095                                     break
3096                 except:
3097                     return None
3098
3099             try:
3100                 #Configure IP profile of the network
3101                 ip_profile = ip_profile if ip_profile is not None else DEFAULT_IP_PROFILE
3102
3103                 if 'subnet_address' not in ip_profile or ip_profile['subnet_address'] is None:
3104                     subnet_rand = random.randint(0, 255)
3105                     ip_base = "192.168.{}.".format(subnet_rand)
3106                     ip_profile['subnet_address'] = ip_base + "0/24"
3107                 else:
3108                     ip_base = ip_profile['subnet_address'].rsplit('.',1)[0] + '.'
3109
3110                 if 'gateway_address' not in ip_profile or ip_profile['gateway_address'] is None:
3111                     ip_profile['gateway_address']=ip_base + "1"
3112                 if 'dhcp_count' not in ip_profile or ip_profile['dhcp_count'] is None:
3113                     ip_profile['dhcp_count']=DEFAULT_IP_PROFILE['dhcp_count']
3114                 if 'dhcp_enabled' not in ip_profile or ip_profile['dhcp_enabled'] is None:
3115                     ip_profile['dhcp_enabled']=DEFAULT_IP_PROFILE['dhcp_enabled']
3116                 if 'dhcp_start_address' not in ip_profile or ip_profile['dhcp_start_address'] is None:
3117                     ip_profile['dhcp_start_address']=ip_base + "3"
3118                 if 'ip_version' not in ip_profile or ip_profile['ip_version'] is None:
3119                     ip_profile['ip_version']=DEFAULT_IP_PROFILE['ip_version']
3120                 if 'dns_address' not in ip_profile or ip_profile['dns_address'] is None:
3121                     ip_profile['dns_address']=ip_base + "2"
3122
3123                 gateway_address=ip_profile['gateway_address']
3124                 dhcp_count=int(ip_profile['dhcp_count'])
3125                 subnet_address=self.convert_cidr_to_netmask(ip_profile['subnet_address'])
3126
3127                 if ip_profile['dhcp_enabled']==True:
3128                     dhcp_enabled='true'
3129                 else:
3130                     dhcp_enabled='false'
3131                 dhcp_start_address=ip_profile['dhcp_start_address']
3132
3133                 #derive dhcp_end_address from dhcp_start_address & dhcp_count
3134                 end_ip_int = int(netaddr.IPAddress(dhcp_start_address))
3135                 end_ip_int += dhcp_count - 1
3136                 dhcp_end_address = str(netaddr.IPAddress(end_ip_int))
3137
3138                 ip_version=ip_profile['ip_version']
3139                 dns_address=ip_profile['dns_address']
3140             except KeyError as exp:
3141                 self.logger.debug("Create Network REST: Key error {}".format(exp))
3142                 raise vimconn.vimconnException("Create Network REST: Key error{}".format(exp))
3143
3144             # either use client provided UUID or search for a first available
3145             #  if both are not defined we return none
3146             if parent_network_uuid is not None:
3147                 url_list = [self.url, '/api/admin/network/', parent_network_uuid]
3148                 add_vdc_rest_url = ''.join(url_list)
3149
3150             #Creating all networks as Direct Org VDC type networks.
3151             #Unused in case of Underlay (data/ptp) network interface.
3152             fence_mode="bridged"
3153             is_inherited='false'
3154             dns_list = dns_address.split(";")
3155             dns1 = dns_list[0]
3156             dns2_text = ""
3157             if len(dns_list) >= 2:
3158                 dns2_text = "\n                                                <Dns2>{}</Dns2>\n".format(dns_list[1])
3159             data = """ <OrgVdcNetwork name="{0:s}" xmlns="http://www.vmware.com/vcloud/v1.5">
3160                             <Description>Openmano created</Description>
3161                                     <Configuration>
3162                                         <IpScopes>
3163                                             <IpScope>
3164                                                 <IsInherited>{1:s}</IsInherited>
3165                                                 <Gateway>{2:s}</Gateway>
3166                                                 <Netmask>{3:s}</Netmask>
3167                                                 <Dns1>{4:s}</Dns1>{5:s}
3168                                                 <IsEnabled>{6:s}</IsEnabled>
3169                                                 <IpRanges>
3170                                                     <IpRange>
3171                                                         <StartAddress>{7:s}</StartAddress>
3172                                                         <EndAddress>{8:s}</EndAddress>
3173                                                     </IpRange>
3174                                                 </IpRanges>
3175                                             </IpScope>
3176                                         </IpScopes>
3177                                         <ParentNetwork href="{9:s}"/>
3178                                         <FenceMode>{10:s}</FenceMode>
3179                                     </Configuration>
3180                                     <IsShared>{11:s}</IsShared>
3181                         </OrgVdcNetwork> """.format(escape(network_name), is_inherited, gateway_address,
3182                                                     subnet_address, dns1, dns2_text, dhcp_enabled,
3183                                                     dhcp_start_address, dhcp_end_address, available_networks,
3184                                                     fence_mode, isshared)
3185
3186             headers['Content-Type'] = 'application/vnd.vmware.vcloud.orgVdcNetwork+xml'
3187             try:
3188                 response = self.perform_request(req_type='POST',
3189                                            url=add_vdc_rest_url,
3190                                            headers=headers,
3191                                            data=data)
3192
3193                 if response.status_code != 201:
3194                     self.logger.debug("Create Network POST REST API call failed. Return status code {}, Response content: {}"
3195                                       .format(response.status_code,response.content))
3196                 else:
3197                     network_task = self.get_task_from_response(response.content)
3198                     self.logger.debug("Create Network REST : Waiting for Network creation complete")
3199                     time.sleep(5)
3200                     result = self.client.get_task_monitor().wait_for_success(task=network_task)
3201                     if result.get('status') == 'success':
3202                         return response.content
3203                     else:
3204                         self.logger.debug("create_network_rest task failed. Network Create response : {}"
3205                                           .format(response.content))
3206             except Exception as exp:
3207                 self.logger.debug("create_network_rest : Exception : {} ".format(exp))
3208
3209         return None
3210
3211     def convert_cidr_to_netmask(self, cidr_ip=None):
3212         """
3213         Method sets convert CIDR netmask address to normal IP format
3214         Args:
3215             cidr_ip : CIDR IP address
3216             Returns:
3217                 netmask : Converted netmask
3218         """
3219         if cidr_ip is not None:
3220             if '/' in cidr_ip:
3221                 network, net_bits = cidr_ip.split('/')
3222                 netmask = socket.inet_ntoa(struct.pack(">I", (0xffffffff << (32 - int(net_bits))) & 0xffffffff))
3223             else:
3224                 netmask = cidr_ip
3225             return netmask
3226         return None
3227
3228     def get_provider_rest(self, vca=None):
3229         """
3230         Method gets provider vdc view from vcloud director
3231
3232         Args:
3233             network_name - is network name to be created.
3234             parent_network_uuid - is parent provider vdc network that will be used for mapping.
3235             It optional attribute. by default if no parent network indicate the first available will be used.
3236
3237             Returns:
3238                 The return xml content of respond or None
3239         """
3240
3241         url_list = [self.url, '/api/admin']
3242         if vca:
3243             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
3244                        'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
3245             response = self.perform_request(req_type='GET',
3246                                             url=''.join(url_list),
3247                                             headers=headers)
3248
3249         if response.status_code == requests.codes.ok:
3250             return response.content
3251         return None
3252
3253     def create_vdc(self, vdc_name=None):
3254
3255         vdc_dict = {}
3256
3257         xml_content = self.create_vdc_from_tmpl_rest(vdc_name=vdc_name)
3258         if xml_content is not None:
3259             try:
3260                 task_resp_xmlroot = XmlElementTree.fromstring(xml_content)
3261                 for child in task_resp_xmlroot:
3262                     if child.tag.split("}")[1] == 'Owner':
3263                         vdc_id = child.attrib.get('href').split("/")[-1]
3264                         vdc_dict[vdc_id] = task_resp_xmlroot.get('href')
3265                         return vdc_dict
3266             except:
3267                 self.logger.debug("Respond body {}".format(xml_content))
3268
3269         return None
3270
3271     def create_vdc_from_tmpl_rest(self, vdc_name=None):
3272         """
3273         Method create vdc in vCloud director based on VDC template.
3274         it uses pre-defined template.
3275
3276         Args:
3277             vdc_name -  name of a new vdc.
3278
3279             Returns:
3280                 The return xml content of respond or None
3281         """
3282         # pre-requesite atleast one vdc template should be available in vCD
3283         self.logger.info("Creating new vdc {}".format(vdc_name))
3284         vca = self.connect_as_admin()
3285         if not vca:
3286             raise vimconn.vimconnConnectionException("Failed to connect vCD")
3287         if vdc_name is None:
3288             return None
3289
3290         url_list = [self.url, '/api/vdcTemplates']
3291         vm_list_rest_call = ''.join(url_list)
3292
3293         headers = {'Accept':'application/*+xml;version=' + API_VERSION,
3294                     'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
3295         response = self.perform_request(req_type='GET',
3296                                         url=vm_list_rest_call,
3297                                         headers=headers)
3298
3299         # container url to a template
3300         vdc_template_ref = None
3301         try:
3302             vm_list_xmlroot = XmlElementTree.fromstring(response.content)
3303             for child in vm_list_xmlroot:
3304                 # application/vnd.vmware.admin.providervdc+xml
3305                 # we need find a template from witch we instantiate VDC
3306                 if child.tag.split("}")[1] == 'VdcTemplate':
3307                     if child.attrib.get('type') == 'application/vnd.vmware.admin.vdcTemplate+xml':
3308                         vdc_template_ref = child.attrib.get('href')
3309         except:
3310             self.logger.debug("Failed parse respond for rest api call {}".format(vm_list_rest_call))
3311             self.logger.debug("Respond body {}".format(response.content))
3312             return None
3313
3314         # if we didn't found required pre defined template we return None
3315         if vdc_template_ref is None:
3316             return None
3317
3318         try:
3319             # instantiate vdc
3320             url_list = [self.url, '/api/org/', self.org_uuid, '/action/instantiate']
3321             vm_list_rest_call = ''.join(url_list)
3322             data = """<InstantiateVdcTemplateParams name="{0:s}" xmlns="http://www.vmware.com/vcloud/v1.5">
3323                                         <Source href="{1:s}"></Source>
3324                                         <Description>opnemano</Description>
3325                                         </InstantiateVdcTemplateParams>""".format(vdc_name, vdc_template_ref)
3326
3327             headers['Content-Type'] = 'application/vnd.vmware.vcloud.instantiateVdcTemplateParams+xml'
3328
3329             response = self.perform_request(req_type='POST',
3330                                             url=vm_list_rest_call,
3331                                             headers=headers,
3332                                             data=data)
3333
3334             vdc_task = self.get_task_from_response(response.content)
3335             self.client.get_task_monitor().wait_for_success(task=vdc_task)
3336
3337             # if we all ok we respond with content otherwise by default None
3338             if response.status_code >= 200 and response.status_code < 300:
3339                 return response.content
3340             return None
3341         except:
3342             self.logger.debug("Failed parse respond for rest api call {}".format(vm_list_rest_call))
3343             self.logger.debug("Respond body {}".format(response.content))
3344
3345         return None
3346
3347     def create_vdc_rest(self, vdc_name=None):
3348         """
3349         Method create network in vCloud director
3350
3351         Args:
3352             vdc_name - vdc name to be created
3353             Returns:
3354                 The return response
3355         """
3356
3357         self.logger.info("Creating new vdc {}".format(vdc_name))
3358
3359         vca = self.connect_as_admin()
3360         if not vca:
3361             raise vimconn.vimconnConnectionException("Failed to connect vCD")
3362         if vdc_name is None:
3363             return None
3364
3365         url_list = [self.url, '/api/admin/org/', self.org_uuid]
3366         vm_list_rest_call = ''.join(url_list)
3367
3368         if vca._session:
3369             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
3370                       'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
3371             response = self.perform_request(req_type='GET',
3372                                             url=vm_list_rest_call,
3373                                             headers=headers)
3374
3375             provider_vdc_ref = None
3376             add_vdc_rest_url = None
3377             available_networks = None
3378
3379             if response.status_code != requests.codes.ok:
3380                 self.logger.debug("REST API call {} failed. Return status code {}".format(vm_list_rest_call,
3381                                                                                           response.status_code))
3382                 return None
3383             else:
3384                 try:
3385                     vm_list_xmlroot = XmlElementTree.fromstring(response.content)
3386                     for child in vm_list_xmlroot:
3387                         # application/vnd.vmware.admin.providervdc+xml
3388                         if child.tag.split("}")[1] == 'Link':
3389                             if child.attrib.get('type') == 'application/vnd.vmware.admin.createVdcParams+xml' \
3390                                     and child.attrib.get('rel') == 'add':
3391                                 add_vdc_rest_url = child.attrib.get('href')
3392                 except:
3393                     self.logger.debug("Failed parse respond for rest api call {}".format(vm_list_rest_call))
3394                     self.logger.debug("Respond body {}".format(response.content))
3395                     return None
3396
3397                 response = self.get_provider_rest(vca=vca)
3398                 try:
3399                     vm_list_xmlroot = XmlElementTree.fromstring(response)
3400                     for child in vm_list_xmlroot:
3401                         if child.tag.split("}")[1] == 'ProviderVdcReferences':
3402                             for sub_child in child:
3403                                 provider_vdc_ref = sub_child.attrib.get('href')
3404                 except:
3405                     self.logger.debug("Failed parse respond for rest api call {}".format(vm_list_rest_call))
3406                     self.logger.debug("Respond body {}".format(response))
3407                     return None
3408
3409                 if add_vdc_rest_url is not None and provider_vdc_ref is not None:
3410                     data = """ <CreateVdcParams name="{0:s}" xmlns="http://www.vmware.com/vcloud/v1.5"><Description>{1:s}</Description>
3411                             <AllocationModel>ReservationPool</AllocationModel>
3412                             <ComputeCapacity><Cpu><Units>MHz</Units><Allocated>2048</Allocated><Limit>2048</Limit></Cpu>
3413                             <Memory><Units>MB</Units><Allocated>2048</Allocated><Limit>2048</Limit></Memory>
3414                             </ComputeCapacity><NicQuota>0</NicQuota><NetworkQuota>100</NetworkQuota>
3415                             <VdcStorageProfile><Enabled>true</Enabled><Units>MB</Units><Limit>20480</Limit><Default>true</Default></VdcStorageProfile>
3416                             <ProviderVdcReference
3417                             name="Main Provider"
3418                             href="{2:s}" />
3419                     <UsesFastProvisioning>true</UsesFastProvisioning></CreateVdcParams>""".format(escape(vdc_name),
3420                                                                                                   escape(vdc_name),
3421                                                                                                   provider_vdc_ref)
3422
3423                     headers['Content-Type'] = 'application/vnd.vmware.admin.createVdcParams+xml'
3424
3425                     response = self.perform_request(req_type='POST',
3426                                                     url=add_vdc_rest_url,
3427                                                     headers=headers,
3428                                                     data=data)
3429
3430                     # if we all ok we respond with content otherwise by default None
3431                     if response.status_code == 201:
3432                         return response.content
3433         return None
3434
3435     def get_vapp_details_rest(self, vapp_uuid=None, need_admin_access=False):
3436         """
3437         Method retrieve vapp detail from vCloud director
3438
3439         Args:
3440             vapp_uuid - is vapp identifier.
3441
3442             Returns:
3443                 The return network uuid or return None
3444         """
3445
3446         parsed_respond = {}
3447         vca = None
3448
3449         if need_admin_access:
3450             vca = self.connect_as_admin()
3451         else:
3452             vca = self.client
3453
3454         if not vca:
3455             raise vimconn.vimconnConnectionException("Failed to connect vCD")
3456         if vapp_uuid is None:
3457             return None
3458
3459         url_list = [self.url, '/api/vApp/vapp-', vapp_uuid]
3460         get_vapp_restcall = ''.join(url_list)
3461
3462         if vca._session:
3463             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
3464                        'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
3465             response = self.perform_request(req_type='GET',
3466                                             url=get_vapp_restcall,
3467                                             headers=headers)
3468
3469             if response.status_code == 403:
3470                 if need_admin_access == False:
3471                     response = self.retry_rest('GET', get_vapp_restcall)
3472
3473             if response.status_code != requests.codes.ok:
3474                 self.logger.debug("REST API call {} failed. Return status code {}".format(get_vapp_restcall,
3475                                                                                           response.status_code))
3476                 return parsed_respond
3477
3478             try:
3479                 xmlroot_respond = XmlElementTree.fromstring(response.content)
3480                 parsed_respond['ovfDescriptorUploaded'] = xmlroot_respond.attrib['ovfDescriptorUploaded']
3481
3482                 namespaces = {"vssd":"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData" ,
3483                               'ovf': 'http://schemas.dmtf.org/ovf/envelope/1',
3484                               'vmw': 'http://www.vmware.com/schema/ovf',
3485                               'vm': 'http://www.vmware.com/vcloud/v1.5',
3486                               'rasd':"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData",
3487                               "vmext":"http://www.vmware.com/vcloud/extension/v1.5",
3488                               "xmlns":"http://www.vmware.com/vcloud/v1.5"
3489                              }
3490
3491                 created_section = xmlroot_respond.find('vm:DateCreated', namespaces)
3492                 if created_section is not None:
3493                     parsed_respond['created'] = created_section.text
3494
3495                 network_section = xmlroot_respond.find('vm:NetworkConfigSection/vm:NetworkConfig', namespaces)
3496                 if network_section is not None and 'networkName' in network_section.attrib:
3497                     parsed_respond['networkname'] = network_section.attrib['networkName']
3498
3499                 ipscopes_section = \
3500                     xmlroot_respond.find('vm:NetworkConfigSection/vm:NetworkConfig/vm:Configuration/vm:IpScopes',
3501                                          namespaces)
3502                 if ipscopes_section is not None:
3503                     for ipscope in ipscopes_section:
3504                         for scope in ipscope:
3505                             tag_key = scope.tag.split("}")[1]
3506                             if tag_key == 'IpRanges':
3507                                 ip_ranges = scope.getchildren()
3508                                 for ipblock in ip_ranges:
3509                                     for block in ipblock:
3510                                         parsed_respond[block.tag.split("}")[1]] = block.text
3511                             else:
3512                                 parsed_respond[tag_key] = scope.text
3513
3514                 # parse children section for other attrib
3515                 children_section = xmlroot_respond.find('vm:Children/', namespaces)
3516                 if children_section is not None:
3517                     parsed_respond['name'] = children_section.attrib['name']
3518                     parsed_respond['nestedHypervisorEnabled'] = children_section.attrib['nestedHypervisorEnabled'] \
3519                      if  "nestedHypervisorEnabled" in children_section.attrib else None
3520                     parsed_respond['deployed'] = children_section.attrib['deployed']
3521                     parsed_respond['status'] = children_section.attrib['status']
3522                     parsed_respond['vmuuid'] = children_section.attrib['id'].split(":")[-1]
3523                     network_adapter = children_section.find('vm:NetworkConnectionSection', namespaces)
3524                     nic_list = []
3525                     for adapters in network_adapter:
3526                         adapter_key = adapters.tag.split("}")[1]
3527                         if adapter_key == 'PrimaryNetworkConnectionIndex':
3528                             parsed_respond['primarynetwork'] = adapters.text
3529                         if adapter_key == 'NetworkConnection':
3530                             vnic = {}
3531                             if 'network' in adapters.attrib:
3532                                 vnic['network'] = adapters.attrib['network']
3533                             for adapter in adapters:
3534                                 setting_key = adapter.tag.split("}")[1]
3535                                 vnic[setting_key] = adapter.text
3536                             nic_list.append(vnic)
3537
3538                     for link in children_section:
3539                         if link.tag.split("}")[1] == 'Link' and 'rel' in link.attrib:
3540                             if link.attrib['rel'] == 'screen:acquireTicket':
3541                                 parsed_respond['acquireTicket'] = link.attrib
3542                             if link.attrib['rel'] == 'screen:acquireMksTicket':
3543                                 parsed_respond['acquireMksTicket'] = link.attrib
3544
3545                     parsed_respond['interfaces'] = nic_list
3546                     vCloud_extension_section = children_section.find('xmlns:VCloudExtension', namespaces)
3547                     if vCloud_extension_section is not None:
3548                         vm_vcenter_info = {}
3549                         vim_info = vCloud_extension_section.find('vmext:VmVimInfo', namespaces)
3550                         vmext = vim_info.find('vmext:VmVimObjectRef', namespaces)
3551                         if vmext is not None:
3552                             vm_vcenter_info["vm_moref_id"] = vmext.find('vmext:MoRef', namespaces).text
3553                         parsed_respond["vm_vcenter_info"]= vm_vcenter_info
3554
3555                     virtual_hardware_section = children_section.find('ovf:VirtualHardwareSection', namespaces)
3556                     vm_virtual_hardware_info = {}
3557                     if virtual_hardware_section is not None:
3558                         for item in virtual_hardware_section.iterfind('ovf:Item',namespaces):
3559                             if item.find("rasd:Description",namespaces).text == "Hard disk":
3560                                 disk_size = item.find("rasd:HostResource" ,namespaces
3561                                                 ).attrib["{"+namespaces['vm']+"}capacity"]
3562
3563                                 vm_virtual_hardware_info["disk_size"]= disk_size
3564                                 break
3565
3566                         for link in virtual_hardware_section:
3567                             if link.tag.split("}")[1] == 'Link' and 'rel' in link.attrib:
3568                                 if link.attrib['rel'] == 'edit' and link.attrib['href'].endswith("/disks"):
3569                                     vm_virtual_hardware_info["disk_edit_href"] = link.attrib['href']
3570                                     break
3571
3572                     parsed_respond["vm_virtual_hardware"]= vm_virtual_hardware_info
3573             except Exception as exp :
3574                 self.logger.info("Error occurred calling rest api for getting vApp details {}".format(exp))
3575         return parsed_respond
3576
3577     def acquire_console(self, vm_uuid=None):
3578
3579         if vm_uuid is None:
3580             return None
3581         if self.client._session:
3582             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
3583                        'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
3584             vm_dict = self.get_vapp_details_rest(vapp_uuid=vm_uuid)
3585             console_dict = vm_dict['acquireTicket']
3586             console_rest_call = console_dict['href']
3587
3588             response = self.perform_request(req_type='POST',
3589                                             url=console_rest_call,
3590                                             headers=headers)
3591
3592             if response.status_code == 403:
3593                 response = self.retry_rest('POST', console_rest_call)
3594
3595             if response.status_code == requests.codes.ok:
3596                 return response.content
3597
3598         return None
3599
3600     def modify_vm_disk(self, vapp_uuid, flavor_disk):
3601         """
3602         Method retrieve vm disk details
3603
3604         Args:
3605             vapp_uuid - is vapp identifier.
3606             flavor_disk - disk size as specified in VNFD (flavor)
3607
3608             Returns:
3609                 The return network uuid or return None
3610         """
3611         status = None
3612         try:
3613             #Flavor disk is in GB convert it into MB
3614             flavor_disk = int(flavor_disk) * 1024
3615             vm_details = self.get_vapp_details_rest(vapp_uuid)
3616             if vm_details:
3617                 vm_name = vm_details["name"]
3618                 self.logger.info("VM: {} flavor_disk :{}".format(vm_name , flavor_disk))
3619
3620             if vm_details and "vm_virtual_hardware" in vm_details:
3621                 vm_disk = int(vm_details["vm_virtual_hardware"]["disk_size"])
3622                 disk_edit_href = vm_details["vm_virtual_hardware"]["disk_edit_href"]
3623
3624                 self.logger.info("VM: {} VM_disk :{}".format(vm_name , vm_disk))
3625
3626                 if flavor_disk > vm_disk:
3627                     status = self.modify_vm_disk_rest(disk_edit_href ,flavor_disk)
3628                     self.logger.info("Modify disk of VM {} from {} to {} MB".format(vm_name,
3629                                                          vm_disk,  flavor_disk ))
3630                 else:
3631                     status = True
3632                     self.logger.info("No need to modify disk of VM {}".format(vm_name))
3633
3634             return status
3635         except Exception as exp:
3636             self.logger.info("Error occurred while modifing disk size {}".format(exp))
3637
3638
3639     def modify_vm_disk_rest(self, disk_href , disk_size):
3640         """
3641         Method retrieve modify vm disk size
3642
3643         Args:
3644             disk_href - vCD API URL to GET and PUT disk data
3645             disk_size - disk size as specified in VNFD (flavor)
3646
3647             Returns:
3648                 The return network uuid or return None
3649         """
3650         if disk_href is None or disk_size is None:
3651             return None
3652
3653         if self.client._session:
3654                 headers = {'Accept':'application/*+xml;version=' + API_VERSION,
3655                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
3656                 response = self.perform_request(req_type='GET',
3657                                                 url=disk_href,
3658                                                 headers=headers)
3659
3660         if response.status_code == 403:
3661             response = self.retry_rest('GET', disk_href)
3662
3663         if response.status_code != requests.codes.ok:
3664             self.logger.debug("GET REST API call {} failed. Return status code {}".format(disk_href,
3665                                                                             response.status_code))
3666             return None
3667         try:
3668             lxmlroot_respond = lxmlElementTree.fromstring(response.content)
3669             namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.iteritems() if prefix}
3670             #For python3
3671             #namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.items() if prefix}
3672             namespaces["xmlns"]= "http://www.vmware.com/vcloud/v1.5"
3673
3674             for item in lxmlroot_respond.iterfind('xmlns:Item',namespaces):
3675                 if item.find("rasd:Description",namespaces).text == "Hard disk":
3676                     disk_item = item.find("rasd:HostResource" ,namespaces )
3677                     if disk_item is not None:
3678                         disk_item.attrib["{"+namespaces['xmlns']+"}capacity"] = str(disk_size)
3679                         break
3680
3681             data = lxmlElementTree.tostring(lxmlroot_respond, encoding='utf8', method='xml',
3682                                              xml_declaration=True)
3683
3684             #Send PUT request to modify disk size
3685             headers['Content-Type'] = 'application/vnd.vmware.vcloud.rasdItemsList+xml; charset=ISO-8859-1'
3686
3687             response = self.perform_request(req_type='PUT',
3688                                                 url=disk_href,
3689                                                 headers=headers,
3690                                                 data=data)
3691             if response.status_code == 403:
3692                 add_headers = {'Content-Type': headers['Content-Type']}
3693                 response = self.retry_rest('PUT', disk_href, add_headers, data)
3694
3695             if response.status_code != 202:
3696                 self.logger.debug("PUT REST API call {} failed. Return status code {}".format(disk_href,
3697                                                                             response.status_code))
3698             else:
3699                 modify_disk_task = self.get_task_from_response(response.content)
3700                 result = self.client.get_task_monitor().wait_for_success(task=modify_disk_task)
3701                 if result.get('status') == 'success':
3702                     return True
3703                 else:
3704                     return False
3705             return None
3706
3707         except Exception as exp :
3708                 self.logger.info("Error occurred calling rest api for modifing disk size {}".format(exp))
3709                 return None
3710
3711     def add_pci_devices(self, vapp_uuid , pci_devices , vmname_andid):
3712         """
3713             Method to attach pci devices to VM
3714
3715              Args:
3716                 vapp_uuid - uuid of vApp/VM
3717                 pci_devices - pci devices infromation as specified in VNFD (flavor)
3718
3719             Returns:
3720                 The status of add pci device task , vm object and
3721                 vcenter_conect object
3722         """
3723         vm_obj = None
3724         self.logger.info("Add pci devices {} into vApp {}".format(pci_devices , vapp_uuid))
3725         vcenter_conect, content = self.get_vcenter_content()
3726         vm_moref_id = self.get_vm_moref_id(vapp_uuid)
3727
3728         if vm_moref_id:
3729             try:
3730                 no_of_pci_devices = len(pci_devices)
3731                 if no_of_pci_devices > 0:
3732                     #Get VM and its host
3733                     host_obj, vm_obj = self.get_vm_obj(content, vm_moref_id)
3734                     self.logger.info("VM {} is currently on host {}".format(vm_obj, host_obj))
3735                     if host_obj and vm_obj:
3736                         #get PCI devies from host on which vapp is currently installed
3737                         avilable_pci_devices = self.get_pci_devices(host_obj, no_of_pci_devices)
3738
3739                         if avilable_pci_devices is None:
3740                             #find other hosts with active pci devices
3741                             new_host_obj , avilable_pci_devices = self.get_host_and_PCIdevices(
3742                                                                 content,
3743                                                                 no_of_pci_devices
3744                                                                 )
3745
3746                             if new_host_obj is not None and avilable_pci_devices is not None and len(avilable_pci_devices)> 0:
3747                                 #Migrate vm to the host where PCI devices are availble
3748                                 self.logger.info("Relocate VM {} on new host {}".format(vm_obj, new_host_obj))
3749                                 task = self.relocate_vm(new_host_obj, vm_obj)
3750                                 if task is not None:
3751                                     result = self.wait_for_vcenter_task(task, vcenter_conect)
3752                                     self.logger.info("Migrate VM status: {}".format(result))
3753                                     host_obj = new_host_obj
3754                                 else:
3755                                     self.logger.info("Fail to migrate VM : {}".format(result))
3756                                     raise vimconn.vimconnNotFoundException(
3757                                     "Fail to migrate VM : {} to host {}".format(
3758                                                     vmname_andid,
3759                                                     new_host_obj)
3760                                         )
3761
3762                         if host_obj is not None and avilable_pci_devices is not None and len(avilable_pci_devices)> 0:
3763                             #Add PCI devices one by one
3764                             for pci_device in avilable_pci_devices:
3765                                 task = self.add_pci_to_vm(host_obj, vm_obj, pci_device)
3766                                 if task:
3767                                     status= self.wait_for_vcenter_task(task, vcenter_conect)
3768                                     if status:
3769                                         self.logger.info("Added PCI device {} to VM {}".format(pci_device,str(vm_obj)))
3770                                 else:
3771                                     self.logger.error("Fail to add PCI device {} to VM {}".format(pci_device,str(vm_obj)))
3772                             return True, vm_obj, vcenter_conect
3773                         else:
3774                             self.logger.error("Currently there is no host with"\
3775                                               " {} number of avaialble PCI devices required for VM {}".format(
3776                                                                             no_of_pci_devices,
3777                                                                             vmname_andid)
3778                                               )
3779                             raise vimconn.vimconnNotFoundException(
3780                                     "Currently there is no host with {} "\
3781                                     "number of avaialble PCI devices required for VM {}".format(
3782                                                                             no_of_pci_devices,
3783                                                                             vmname_andid))
3784                 else:
3785                     self.logger.debug("No infromation about PCI devices {} ",pci_devices)
3786
3787             except vmodl.MethodFault as error:
3788                 self.logger.error("Error occurred while adding PCI devices {} ",error)
3789         return None, vm_obj, vcenter_conect
3790
3791     def get_vm_obj(self, content, mob_id):
3792         """
3793             Method to get the vsphere VM object associated with a given morf ID
3794              Args:
3795                 vapp_uuid - uuid of vApp/VM
3796                 content - vCenter content object
3797                 mob_id - mob_id of VM
3798
3799             Returns:
3800                     VM and host object
3801         """
3802         vm_obj = None
3803         host_obj = None
3804         try :
3805             container = content.viewManager.CreateContainerView(content.rootFolder,
3806                                                         [vim.VirtualMachine], True
3807                                                         )
3808             for vm in container.view:
3809                 mobID = vm._GetMoId()
3810                 if mobID == mob_id:
3811                     vm_obj = vm
3812                     host_obj = vm_obj.runtime.host
3813                     break
3814         except Exception as exp:
3815             self.logger.error("Error occurred while finding VM object : {}".format(exp))
3816         return host_obj, vm_obj
3817
3818     def get_pci_devices(self, host, need_devices):
3819         """
3820             Method to get the details of pci devices on given host
3821              Args:
3822                 host - vSphere host object
3823                 need_devices - number of pci devices needed on host
3824
3825              Returns:
3826                 array of pci devices
3827         """
3828         all_devices = []
3829         all_device_ids = []
3830         used_devices_ids = []
3831
3832         try:
3833             if host:
3834                 pciPassthruInfo = host.config.pciPassthruInfo
3835                 pciDevies = host.hardware.pciDevice
3836
3837             for pci_status in pciPassthruInfo:
3838                 if pci_status.passthruActive:
3839                     for device in pciDevies:
3840                         if device.id == pci_status.id:
3841                             all_device_ids.append(device.id)
3842                             all_devices.append(device)
3843
3844             #check if devices are in use
3845             avalible_devices = all_devices
3846             for vm in host.vm:
3847                 if vm.runtime.powerState == vim.VirtualMachinePowerState.poweredOn:
3848                     vm_devices = vm.config.hardware.device
3849                     for device in vm_devices:
3850                         if type(device) is vim.vm.device.VirtualPCIPassthrough:
3851                             if device.backing.id in all_device_ids:
3852                                 for use_device in avalible_devices:
3853                                     if use_device.id == device.backing.id:
3854                                         avalible_devices.remove(use_device)
3855                                 used_devices_ids.append(device.backing.id)
3856                                 self.logger.debug("Device {} from devices {}"\
3857                                         "is in use".format(device.backing.id,
3858                                                            device)
3859                                             )
3860             if len(avalible_devices) < need_devices:
3861                 self.logger.debug("Host {} don't have {} number of active devices".format(host,
3862                                                                             need_devices))
3863                 self.logger.debug("found only {} devives {}".format(len(avalible_devices),
3864                                                                     avalible_devices))
3865                 return None
3866             else:
3867                 required_devices = avalible_devices[:need_devices]
3868                 self.logger.info("Found {} PCI devivces on host {} but required only {}".format(
3869                                                             len(avalible_devices),
3870                                                             host,
3871                                                             need_devices))
3872                 self.logger.info("Retruning {} devices as {}".format(need_devices,
3873                                                                 required_devices ))
3874                 return required_devices
3875
3876         except Exception as exp:
3877             self.logger.error("Error {} occurred while finding pci devices on host: {}".format(exp, host))
3878
3879         return None
3880
3881     def get_host_and_PCIdevices(self, content, need_devices):
3882         """
3883          Method to get the details of pci devices infromation on all hosts
3884
3885             Args:
3886                 content - vSphere host object
3887                 need_devices - number of pci devices needed on host
3888
3889             Returns:
3890                  array of pci devices and host object
3891         """
3892         host_obj = None
3893         pci_device_objs = None
3894         try:
3895             if content:
3896                 container = content.viewManager.CreateContainerView(content.rootFolder,
3897                                                             [vim.HostSystem], True)
3898                 for host in container.view:
3899                     devices = self.get_pci_devices(host, need_devices)
3900                     if devices:
3901                         host_obj = host
3902                         pci_device_objs = devices
3903                         break
3904         except Exception as exp:
3905             self.logger.error("Error {} occurred while finding pci devices on host: {}".format(exp, host_obj))
3906
3907         return host_obj,pci_device_objs
3908
3909     def relocate_vm(self, dest_host, vm) :
3910         """
3911          Method to get the relocate VM to new host
3912
3913             Args:
3914                 dest_host - vSphere host object
3915                 vm - vSphere VM object
3916
3917             Returns:
3918                 task object
3919         """
3920         task = None
3921         try:
3922             relocate_spec = vim.vm.RelocateSpec(host=dest_host)
3923             task = vm.Relocate(relocate_spec)
3924             self.logger.info("Migrating {} to destination host {}".format(vm, dest_host))
3925         except Exception as exp:
3926             self.logger.error("Error occurred while relocate VM {} to new host {}: {}".format(
3927                                                                             dest_host, vm, exp))
3928         return task
3929
3930     def wait_for_vcenter_task(self, task, actionName='job', hideResult=False):
3931         """
3932         Waits and provides updates on a vSphere task
3933         """
3934         while task.info.state == vim.TaskInfo.State.running:
3935             time.sleep(2)
3936
3937         if task.info.state == vim.TaskInfo.State.success:
3938             if task.info.result is not None and not hideResult:
3939                 self.logger.info('{} completed successfully, result: {}'.format(
3940                                                             actionName,
3941                                                             task.info.result))
3942             else:
3943                 self.logger.info('Task {} completed successfully.'.format(actionName))
3944         else:
3945             self.logger.error('{} did not complete successfully: {} '.format(
3946                                                             actionName,
3947                                                             task.info.error)
3948                               )
3949
3950         return task.info.result
3951
3952     def add_pci_to_vm(self,host_object, vm_object, host_pci_dev):
3953         """
3954          Method to add pci device in given VM
3955
3956             Args:
3957                 host_object - vSphere host object
3958                 vm_object - vSphere VM object
3959                 host_pci_dev -  host_pci_dev must be one of the devices from the
3960                                 host_object.hardware.pciDevice list
3961                                 which is configured as a PCI passthrough device
3962
3963             Returns:
3964                 task object
3965         """
3966         task = None
3967         if vm_object and host_object and host_pci_dev:
3968             try :
3969                 #Add PCI device to VM
3970                 pci_passthroughs = vm_object.environmentBrowser.QueryConfigTarget(host=None).pciPassthrough
3971                 systemid_by_pciid = {item.pciDevice.id: item.systemId for item in pci_passthroughs}
3972
3973                 if host_pci_dev.id not in systemid_by_pciid:
3974                     self.logger.error("Device {} is not a passthrough device ".format(host_pci_dev))
3975                     return None
3976
3977                 deviceId = hex(host_pci_dev.deviceId % 2**16).lstrip('0x')
3978                 backing = vim.VirtualPCIPassthroughDeviceBackingInfo(deviceId=deviceId,
3979                                             id=host_pci_dev.id,
3980                                             systemId=systemid_by_pciid[host_pci_dev.id],
3981                                             vendorId=host_pci_dev.vendorId,
3982                                             deviceName=host_pci_dev.deviceName)
3983
3984                 hba_object = vim.VirtualPCIPassthrough(key=-100, backing=backing)
3985
3986                 new_device_config = vim.VirtualDeviceConfigSpec(device=hba_object)
3987                 new_device_config.operation = "add"
3988                 vmConfigSpec = vim.vm.ConfigSpec()
3989                 vmConfigSpec.deviceChange = [new_device_config]
3990
3991                 task = vm_object.ReconfigVM_Task(spec=vmConfigSpec)
3992                 self.logger.info("Adding PCI device {} into VM {} from host {} ".format(
3993                                                             host_pci_dev, vm_object, host_object)
3994                                 )
3995             except Exception as exp:
3996                 self.logger.error("Error occurred while adding pci devive {} to VM {}: {}".format(
3997                                                                             host_pci_dev,
3998                                                                             vm_object,
3999                                                                              exp))
4000         return task
4001
4002     def get_vm_vcenter_info(self):
4003         """
4004         Method to get details of vCenter and vm
4005
4006             Args:
4007                 vapp_uuid - uuid of vApp or VM
4008
4009             Returns:
4010                 Moref Id of VM and deails of vCenter
4011         """
4012         vm_vcenter_info = {}
4013
4014         if self.vcenter_ip is not None:
4015             vm_vcenter_info["vm_vcenter_ip"] = self.vcenter_ip
4016         else:
4017             raise vimconn.vimconnException(message="vCenter IP is not provided."\
4018                                            " Please provide vCenter IP while attaching datacenter to tenant in --config")
4019         if self.vcenter_port is not None:
4020             vm_vcenter_info["vm_vcenter_port"] = self.vcenter_port
4021         else:
4022             raise vimconn.vimconnException(message="vCenter port is not provided."\
4023                                            " Please provide vCenter port while attaching datacenter to tenant in --config")
4024         if self.vcenter_user is not None:
4025             vm_vcenter_info["vm_vcenter_user"] = self.vcenter_user
4026         else:
4027             raise vimconn.vimconnException(message="vCenter user is not provided."\
4028                                            " Please provide vCenter user while attaching datacenter to tenant in --config")
4029
4030         if self.vcenter_password is not None:
4031             vm_vcenter_info["vm_vcenter_password"] = self.vcenter_password
4032         else:
4033             raise vimconn.vimconnException(message="vCenter user password is not provided."\
4034                                            " Please provide vCenter user password while attaching datacenter to tenant in --config")
4035
4036         return vm_vcenter_info
4037
4038
4039     def get_vm_pci_details(self, vmuuid):
4040         """
4041             Method to get VM PCI device details from vCenter
4042
4043             Args:
4044                 vm_obj - vSphere VM object
4045
4046             Returns:
4047                 dict of PCI devives attached to VM
4048
4049         """
4050         vm_pci_devices_info = {}
4051         try:
4052             vcenter_conect, content = self.get_vcenter_content()
4053             vm_moref_id = self.get_vm_moref_id(vmuuid)
4054             if vm_moref_id:
4055                 #Get VM and its host
4056                 if content:
4057                     host_obj, vm_obj = self.get_vm_obj(content, vm_moref_id)
4058                     if host_obj and vm_obj:
4059                         vm_pci_devices_info["host_name"]= host_obj.name
4060                         vm_pci_devices_info["host_ip"]= host_obj.config.network.vnic[0].spec.ip.ipAddress
4061                         for device in vm_obj.config.hardware.device:
4062                             if type(device) == vim.vm.device.VirtualPCIPassthrough:
4063                                 device_details={'devide_id':device.backing.id,
4064                                                 'pciSlotNumber':device.slotInfo.pciSlotNumber,
4065                                             }
4066                                 vm_pci_devices_info[device.deviceInfo.label] = device_details
4067                 else:
4068                     self.logger.error("Can not connect to vCenter while getting "\
4069                                           "PCI devices infromationn")
4070                 return vm_pci_devices_info
4071         except Exception as exp:
4072             self.logger.error("Error occurred while getting VM infromationn"\
4073                              " for VM : {}".format(exp))
4074             raise vimconn.vimconnException(message=exp)
4075
4076     def add_network_adapter_to_vms(self, vapp, network_name, primary_nic_index, nicIndex, net, nic_type=None):
4077         """
4078             Method to add network adapter type to vm
4079             Args :
4080                 network_name - name of network
4081                 primary_nic_index - int value for primary nic index
4082                 nicIndex - int value for nic index
4083                 nic_type - specify model name to which add to vm
4084             Returns:
4085                 None
4086         """
4087
4088         try:
4089             ip_address = None
4090             floating_ip = False
4091             mac_address = None
4092             if 'floating_ip' in net: floating_ip = net['floating_ip']
4093
4094             # Stub for ip_address feature
4095             if 'ip_address' in net: ip_address = net['ip_address']
4096
4097             if 'mac_address' in net: mac_address = net['mac_address']
4098
4099             if floating_ip:
4100                 allocation_mode = "POOL"
4101             elif ip_address:
4102                 allocation_mode = "MANUAL"
4103             else:
4104                 allocation_mode = "DHCP"
4105
4106             if not nic_type:
4107                 for vms in vapp.get_all_vms():
4108                     vm_id = vms.get('id').split(':')[-1]
4109
4110                     url_rest_call = "{}/api/vApp/vm-{}/networkConnectionSection/".format(self.url, vm_id)
4111
4112                     headers = {'Accept':'application/*+xml;version=' + API_VERSION,
4113                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
4114                     response = self.perform_request(req_type='GET',
4115                                                     url=url_rest_call,
4116                                                     headers=headers)
4117
4118                     if response.status_code == 403:
4119                         response = self.retry_rest('GET', url_rest_call)
4120
4121                     if response.status_code != 200:
4122                         self.logger.error("REST call {} failed reason : {}"\
4123                                              "status code : {}".format(url_rest_call,
4124                                                                     response.content,
4125                                                                response.status_code))
4126                         raise vimconn.vimconnException("add_network_adapter_to_vms : Failed to get "\
4127                                                                          "network connection section")
4128
4129                     data = response.content
4130                     data = data.split('<Link rel="edit"')[0]
4131                     if '<PrimaryNetworkConnectionIndex>' not in data:
4132                         item = """<PrimaryNetworkConnectionIndex>{}</PrimaryNetworkConnectionIndex>
4133                                 <NetworkConnection network="{}">
4134                                 <NetworkConnectionIndex>{}</NetworkConnectionIndex>
4135                                 <IsConnected>true</IsConnected>
4136                                 <IpAddressAllocationMode>{}</IpAddressAllocationMode>
4137                                 </NetworkConnection>""".format(primary_nic_index, network_name, nicIndex,
4138                                                                                          allocation_mode)
4139                         # Stub for ip_address feature
4140                         if ip_address:
4141                             ip_tag = '<IpAddress>{}</IpAddress>'.format(ip_address)
4142                             item =  item.replace('</NetworkConnectionIndex>\n','</NetworkConnectionIndex>\n{}\n'.format(ip_tag))
4143
4144                         if mac_address:
4145                             mac_tag = '<MACAddress>{}</MACAddress>'.format(mac_address)
4146                             item =  item.replace('</IsConnected>\n','</IsConnected>\n{}\n'.format(mac_tag))
4147
4148                         data = data.replace('</ovf:Info>\n','</ovf:Info>\n{}\n</NetworkConnectionSection>'.format(item))
4149                     else:
4150                         new_item = """<NetworkConnection network="{}">
4151                                     <NetworkConnectionIndex>{}</NetworkConnectionIndex>
4152                                     <IsConnected>true</IsConnected>
4153                                     <IpAddressAllocationMode>{}</IpAddressAllocationMode>
4154                                     </NetworkConnection>""".format(network_name, nicIndex,
4155                                                                           allocation_mode)
4156                         # Stub for ip_address feature
4157                         if ip_address:
4158                             ip_tag = '<IpAddress>{}</IpAddress>'.format(ip_address)
4159                             new_item =  new_item.replace('</NetworkConnectionIndex>\n','</NetworkConnectionIndex>\n{}\n'.format(ip_tag))
4160
4161                         if mac_address:
4162                             mac_tag = '<MACAddress>{}</MACAddress>'.format(mac_address)
4163                             new_item =  new_item.replace('</IsConnected>\n','</IsConnected>\n{}\n'.format(mac_tag))
4164
4165                         data = data + new_item + '</NetworkConnectionSection>'
4166
4167                     headers['Content-Type'] = 'application/vnd.vmware.vcloud.networkConnectionSection+xml'
4168
4169                     response = self.perform_request(req_type='PUT',
4170                                                     url=url_rest_call,
4171                                                     headers=headers,
4172                                                     data=data)
4173
4174                     if response.status_code == 403:
4175                         add_headers = {'Content-Type': headers['Content-Type']}
4176                         response = self.retry_rest('PUT', url_rest_call, add_headers, data)
4177
4178                     if response.status_code != 202:
4179                         self.logger.error("REST call {} failed reason : {}"\
4180                                             "status code : {} ".format(url_rest_call,
4181                                                                     response.content,
4182                                                                response.status_code))
4183                         raise vimconn.vimconnException("add_network_adapter_to_vms : Failed to update "\
4184                                                                             "network connection section")
4185                     else:
4186                         nic_task = self.get_task_from_response(response.content)
4187                         result = self.client.get_task_monitor().wait_for_success(task=nic_task)
4188                         if result.get('status') == 'success':
4189                             self.logger.info("add_network_adapter_to_vms(): VM {} conneced to "\
4190                                                                "default NIC type".format(vm_id))
4191                         else:
4192                             self.logger.error("add_network_adapter_to_vms(): VM {} failed to "\
4193                                                               "connect NIC type".format(vm_id))
4194             else:
4195                 for vms in vapp.get_all_vms():
4196                     vm_id = vms.get('id').split(':')[-1]
4197
4198
4199                     url_rest_call = "{}/api/vApp/vm-{}/networkConnectionSection/".format(self.url, vm_id)
4200
4201                     headers = {'Accept':'application/*+xml;version=' + API_VERSION,
4202                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
4203                     response = self.perform_request(req_type='GET',
4204                                                     url=url_rest_call,
4205                                                     headers=headers)
4206
4207                     if response.status_code == 403:
4208                         response = self.retry_rest('GET', url_rest_call)
4209
4210                     if response.status_code != 200:
4211                         self.logger.error("REST call {} failed reason : {}"\
4212                                             "status code : {}".format(url_rest_call,
4213                                                                    response.content,
4214                                                               response.status_code))
4215                         raise vimconn.vimconnException("add_network_adapter_to_vms : Failed to get "\
4216                                                                         "network connection section")
4217                     data = response.content
4218                     data = data.split('<Link rel="edit"')[0]
4219                     if '<PrimaryNetworkConnectionIndex>' not in data:
4220                         item = """<PrimaryNetworkConnectionIndex>{}</PrimaryNetworkConnectionIndex>
4221                                 <NetworkConnection network="{}">
4222                                 <NetworkConnectionIndex>{}</NetworkConnectionIndex>
4223                                 <IsConnected>true</IsConnected>
4224                                 <IpAddressAllocationMode>{}</IpAddressAllocationMode>
4225                                 <NetworkAdapterType>{}</NetworkAdapterType>
4226                                 </NetworkConnection>""".format(primary_nic_index, network_name, nicIndex,
4227                                                                                allocation_mode, nic_type)
4228                         # Stub for ip_address feature
4229                         if ip_address:
4230                             ip_tag = '<IpAddress>{}</IpAddress>'.format(ip_address)
4231                             item =  item.replace('</NetworkConnectionIndex>\n','</NetworkConnectionIndex>\n{}\n'.format(ip_tag))
4232
4233                         if mac_address:
4234                             mac_tag = '<MACAddress>{}</MACAddress>'.format(mac_address)
4235                             item =  item.replace('</IsConnected>\n','</IsConnected>\n{}\n'.format(mac_tag))
4236
4237                         data = data.replace('</ovf:Info>\n','</ovf:Info>\n{}\n</NetworkConnectionSection>'.format(item))
4238                     else:
4239                         new_item = """<NetworkConnection network="{}">
4240                                     <NetworkConnectionIndex>{}</NetworkConnectionIndex>
4241                                     <IsConnected>true</IsConnected>
4242                                     <IpAddressAllocationMode>{}</IpAddressAllocationMode>
4243                                     <NetworkAdapterType>{}</NetworkAdapterType>
4244                                     </NetworkConnection>""".format(network_name, nicIndex,
4245                                                                 allocation_mode, nic_type)
4246                         # Stub for ip_address feature
4247                         if ip_address:
4248                             ip_tag = '<IpAddress>{}</IpAddress>'.format(ip_address)
4249                             new_item =  new_item.replace('</NetworkConnectionIndex>\n','</NetworkConnectionIndex>\n{}\n'.format(ip_tag))
4250
4251                         if mac_address:
4252                             mac_tag = '<MACAddress>{}</MACAddress>'.format(mac_address)
4253                             new_item =  new_item.replace('</IsConnected>\n','</IsConnected>\n{}\n'.format(mac_tag))
4254
4255                         data = data + new_item + '</NetworkConnectionSection>'
4256
4257                     headers['Content-Type'] = 'application/vnd.vmware.vcloud.networkConnectionSection+xml'
4258
4259                     response = self.perform_request(req_type='PUT',
4260                                                     url=url_rest_call,
4261                                                     headers=headers,
4262                                                     data=data)
4263
4264                     if response.status_code == 403:
4265                         add_headers = {'Content-Type': headers['Content-Type']}
4266                         response = self.retry_rest('PUT', url_rest_call, add_headers, data)
4267
4268                     if response.status_code != 202:
4269                         self.logger.error("REST call {} failed reason : {}"\
4270                                             "status code : {}".format(url_rest_call,
4271                                                                    response.content,
4272                                                               response.status_code))
4273                         raise vimconn.vimconnException("add_network_adapter_to_vms : Failed to update "\
4274                                                                            "network connection section")
4275                     else:
4276                         nic_task = self.get_task_from_response(response.content)
4277                         result = self.client.get_task_monitor().wait_for_success(task=nic_task)
4278                         if result.get('status') == 'success':
4279                             self.logger.info("add_network_adapter_to_vms(): VM {} "\
4280                                                "conneced to NIC type {}".format(vm_id, nic_type))
4281                         else:
4282                             self.logger.error("add_network_adapter_to_vms(): VM {} "\
4283                                                "failed to connect NIC type {}".format(vm_id, nic_type))
4284         except Exception as exp:
4285             self.logger.error("add_network_adapter_to_vms() : exception occurred "\
4286                                                "while adding Network adapter")
4287             raise vimconn.vimconnException(message=exp)
4288
4289
4290     def set_numa_affinity(self, vmuuid, paired_threads_id):
4291         """
4292             Method to assign numa affinity in vm configuration parammeters
4293             Args :
4294                 vmuuid - vm uuid
4295                 paired_threads_id - one or more virtual processor
4296                                     numbers
4297             Returns:
4298                 return if True
4299         """
4300         try:
4301             vcenter_conect, content = self.get_vcenter_content()
4302             vm_moref_id = self.get_vm_moref_id(vmuuid)
4303
4304             host_obj, vm_obj = self.get_vm_obj(content ,vm_moref_id)
4305             if vm_obj:
4306                 config_spec = vim.vm.ConfigSpec()
4307                 config_spec.extraConfig = []
4308                 opt = vim.option.OptionValue()
4309                 opt.key = 'numa.nodeAffinity'
4310                 opt.value = str(paired_threads_id)
4311                 config_spec.extraConfig.append(opt)
4312                 task = vm_obj.ReconfigVM_Task(config_spec)
4313                 if task:
4314                     result = self.wait_for_vcenter_task(task, vcenter_conect)
4315                     extra_config = vm_obj.config.extraConfig
4316                     flag = False
4317                     for opts in extra_config:
4318                         if 'numa.nodeAffinity' in opts.key:
4319                             flag = True
4320                             self.logger.info("set_numa_affinity: Sucessfully assign numa affinity "\
4321                                                      "value {} for vm {}".format(opt.value, vm_obj))
4322                         if flag:
4323                             return
4324             else:
4325                 self.logger.error("set_numa_affinity: Failed to assign numa affinity")
4326         except Exception as exp:
4327             self.logger.error("set_numa_affinity : exception occurred while setting numa affinity "\
4328                                                        "for VM {} : {}".format(vm_obj, vm_moref_id))
4329             raise vimconn.vimconnException("set_numa_affinity : Error {} failed to assign numa "\
4330                                                                            "affinity".format(exp))
4331
4332
4333     def cloud_init(self, vapp, cloud_config):
4334         """
4335         Method to inject ssh-key
4336         vapp - vapp object
4337         cloud_config a dictionary with:
4338                 'key-pairs': (optional) list of strings with the public key to be inserted to the default user
4339                 'users': (optional) list of users to be inserted, each item is a dict with:
4340                     'name': (mandatory) user name,
4341                     'key-pairs': (optional) list of strings with the public key to be inserted to the user
4342                 'user-data': (optional) can be a string with the text script to be passed directly to cloud-init,
4343                     or a list of strings, each one contains a script to be passed, usually with a MIMEmultipart file
4344                 'config-files': (optional). List of files to be transferred. Each item is a dict with:
4345                     'dest': (mandatory) string with the destination absolute path
4346                     'encoding': (optional, by default text). Can be one of:
4347                         'b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64'
4348                     'content' (mandatory): string with the content of the file
4349                     'permissions': (optional) string with file permissions, typically octal notation '0644'
4350                     'owner': (optional) file owner, string with the format 'owner:group'
4351                 'boot-data-drive': boolean to indicate if user-data must be passed using a boot drive (hard disk
4352         """
4353         try:
4354             if not isinstance(cloud_config, dict):
4355                 raise Exception("cloud_init : parameter cloud_config is not a dictionary")
4356             else:
4357                 key_pairs = []
4358                 userdata = []
4359                 if "key-pairs" in cloud_config:
4360                     key_pairs = cloud_config["key-pairs"]
4361
4362                 if "users" in cloud_config:
4363                     userdata = cloud_config["users"]
4364
4365                 self.logger.debug("cloud_init : Guest os customization started..")
4366                 customize_script = self.format_script(key_pairs=key_pairs, users_list=userdata)
4367                 customize_script = customize_script.replace("&","&amp;")
4368                 self.guest_customization(vapp, customize_script)
4369
4370         except Exception as exp:
4371             self.logger.error("cloud_init : exception occurred while injecting "\
4372                                                                        "ssh-key")
4373             raise vimconn.vimconnException("cloud_init : Error {} failed to inject "\
4374                                                                "ssh-key".format(exp))
4375
4376     def format_script(self, key_pairs=[], users_list=[]):
4377         bash_script = """#!/bin/sh
4378         echo performing customization tasks with param $1 at `date "+DATE: %Y-%m-%d - TIME: %H:%M:%S"` >> /root/customization.log
4379         if [ "$1" = "precustomization" ];then
4380             echo performing precustomization tasks   on `date "+DATE: %Y-%m-%d - TIME: %H:%M:%S"` >> /root/customization.log
4381         """
4382
4383         keys = "\n".join(key_pairs)
4384         if keys:
4385             keys_data = """
4386             if [ ! -d /root/.ssh ];then
4387                 mkdir /root/.ssh
4388                 chown root:root /root/.ssh
4389                 chmod 700 /root/.ssh
4390                 touch /root/.ssh/authorized_keys
4391                 chown root:root /root/.ssh/authorized_keys
4392                 chmod 600 /root/.ssh/authorized_keys
4393                 # make centos with selinux happy
4394                 which restorecon && restorecon -Rv /root/.ssh
4395             else
4396                 touch /root/.ssh/authorized_keys
4397                 chown root:root /root/.ssh/authorized_keys
4398                 chmod 600 /root/.ssh/authorized_keys
4399             fi
4400             echo '{key}' >> /root/.ssh/authorized_keys
4401             """.format(key=keys)
4402
4403             bash_script+= keys_data
4404
4405         for user in users_list:
4406             if 'name' in user: user_name = user['name']
4407             if 'key-pairs' in user:
4408                 user_keys = "\n".join(user['key-pairs'])
4409             else:
4410                 user_keys = None
4411
4412             add_user_name = """
4413                 useradd -d /home/{user_name} -m -g users -s /bin/bash {user_name}
4414                 """.format(user_name=user_name)
4415
4416             bash_script+= add_user_name
4417
4418             if user_keys:
4419                 user_keys_data = """
4420                 mkdir /home/{user_name}/.ssh
4421                 chown {user_name}:{user_name} /home/{user_name}/.ssh
4422                 chmod 700 /home/{user_name}/.ssh
4423                 touch /home/{user_name}/.ssh/authorized_keys
4424                 chown {user_name}:{user_name} /home/{user_name}/.ssh/authorized_keys
4425                 chmod 600 /home/{user_name}/.ssh/authorized_keys
4426                 # make centos with selinux happy
4427                 which restorecon && restorecon -Rv /home/{user_name}/.ssh
4428                 echo '{user_key}' >> /home/{user_name}/.ssh/authorized_keys
4429                 """.format(user_name=user_name,user_key=user_keys)
4430
4431                 bash_script+= user_keys_data
4432
4433         return bash_script+"\n\tfi"
4434
4435     def guest_customization(self, vapp, customize_script):
4436         """
4437         Method to customize guest os
4438         vapp - Vapp object
4439         customize_script - Customize script to be run at first boot of VM.
4440         """
4441         for vm in vapp.get_all_vms():
4442             vm_id = vm.get('id').split(':')[-1]
4443             vm_name = vm.get('name')
4444             vm_name = vm_name.replace('_','-')
4445
4446             vm_customization_url = "{}/api/vApp/vm-{}/guestCustomizationSection/".format(self.url, vm_id)
4447             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
4448                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
4449
4450             headers['Content-Type'] = "application/vnd.vmware.vcloud.guestCustomizationSection+xml"
4451
4452             data = """<GuestCustomizationSection
4453                            xmlns="http://www.vmware.com/vcloud/v1.5"
4454                            xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/1"
4455                            ovf:required="false" href="{}" type="application/vnd.vmware.vcloud.guestCustomizationSection+xml">
4456                            <ovf:Info>Specifies Guest OS Customization Settings</ovf:Info>
4457                            <Enabled>true</Enabled>
4458                            <ChangeSid>false</ChangeSid>
4459                            <VirtualMachineId>{}</VirtualMachineId>
4460                            <JoinDomainEnabled>false</JoinDomainEnabled>
4461                            <UseOrgSettings>false</UseOrgSettings>
4462                            <AdminPasswordEnabled>false</AdminPasswordEnabled>
4463                            <AdminPasswordAuto>true</AdminPasswordAuto>
4464                            <AdminAutoLogonEnabled>false</AdminAutoLogonEnabled>
4465                            <AdminAutoLogonCount>0</AdminAutoLogonCount>
4466                            <ResetPasswordRequired>false</ResetPasswordRequired>
4467                            <CustomizationScript>{}</CustomizationScript>
4468                            <ComputerName>{}</ComputerName>
4469                            <Link href="{}" type="application/vnd.vmware.vcloud.guestCustomizationSection+xml" rel="edit"/>
4470                        </GuestCustomizationSection>
4471                    """.format(vm_customization_url,
4472                                              vm_id,
4473                                   customize_script,
4474                                            vm_name,
4475                               vm_customization_url)
4476
4477             response = self.perform_request(req_type='PUT',
4478                                              url=vm_customization_url,
4479                                              headers=headers,
4480                                              data=data)
4481             if response.status_code == 202:
4482                 guest_task = self.get_task_from_response(response.content)
4483                 self.client.get_task_monitor().wait_for_success(task=guest_task)
4484                 self.logger.info("guest_customization : customized guest os task "\
4485                                              "completed for VM {}".format(vm_name))
4486             else:
4487                 self.logger.error("guest_customization : task for customized guest os"\
4488                                                     "failed for VM {}".format(vm_name))
4489                 raise vimconn.vimconnException("guest_customization : failed to perform"\
4490                                        "guest os customization on VM {}".format(vm_name))
4491
4492     def add_new_disk(self, vapp_uuid, disk_size):
4493         """
4494             Method to create an empty vm disk
4495
4496             Args:
4497                 vapp_uuid - is vapp identifier.
4498                 disk_size - size of disk to be created in GB
4499
4500             Returns:
4501                 None
4502         """
4503         status = False
4504         vm_details = None
4505         try:
4506             #Disk size in GB, convert it into MB
4507             if disk_size is not None:
4508                 disk_size_mb = int(disk_size) * 1024
4509                 vm_details = self.get_vapp_details_rest(vapp_uuid)
4510
4511             if vm_details and "vm_virtual_hardware" in vm_details:
4512                 self.logger.info("Adding disk to VM: {} disk size:{}GB".format(vm_details["name"], disk_size))
4513                 disk_href = vm_details["vm_virtual_hardware"]["disk_edit_href"]
4514                 status = self.add_new_disk_rest(disk_href, disk_size_mb)
4515
4516         except Exception as exp:
4517             msg = "Error occurred while creating new disk {}.".format(exp)
4518             self.rollback_newvm(vapp_uuid, msg)
4519
4520         if status:
4521             self.logger.info("Added new disk to VM: {} disk size:{}GB".format(vm_details["name"], disk_size))
4522         else:
4523             #If failed to add disk, delete VM
4524             msg = "add_new_disk: Failed to add new disk to {}".format(vm_details["name"])
4525             self.rollback_newvm(vapp_uuid, msg)
4526
4527
4528     def add_new_disk_rest(self, disk_href, disk_size_mb):
4529         """
4530         Retrives vApp Disks section & add new empty disk
4531
4532         Args:
4533             disk_href: Disk section href to addd disk
4534             disk_size_mb: Disk size in MB
4535
4536             Returns: Status of add new disk task
4537         """
4538         status = False
4539         if self.client._session:
4540             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
4541                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
4542             response = self.perform_request(req_type='GET',
4543                                             url=disk_href,
4544                                             headers=headers)
4545
4546         if response.status_code == 403:
4547             response = self.retry_rest('GET', disk_href)
4548
4549         if response.status_code != requests.codes.ok:
4550             self.logger.error("add_new_disk_rest: GET REST API call {} failed. Return status code {}"
4551                               .format(disk_href, response.status_code))
4552             return status
4553         try:
4554             #Find but type & max of instance IDs assigned to disks
4555             lxmlroot_respond = lxmlElementTree.fromstring(response.content)
4556             namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.iteritems() if prefix}
4557             #For python3
4558             #namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.items() if prefix}
4559             namespaces["xmlns"]= "http://www.vmware.com/vcloud/v1.5"
4560             instance_id = 0
4561             for item in lxmlroot_respond.iterfind('xmlns:Item',namespaces):
4562                 if item.find("rasd:Description",namespaces).text == "Hard disk":
4563                     inst_id = int(item.find("rasd:InstanceID" ,namespaces).text)
4564                     if inst_id > instance_id:
4565                         instance_id = inst_id
4566                         disk_item = item.find("rasd:HostResource" ,namespaces)
4567                         bus_subtype = disk_item.attrib["{"+namespaces['xmlns']+"}busSubType"]
4568                         bus_type = disk_item.attrib["{"+namespaces['xmlns']+"}busType"]
4569
4570             instance_id = instance_id + 1
4571             new_item =   """<Item>
4572                                 <rasd:Description>Hard disk</rasd:Description>
4573                                 <rasd:ElementName>New disk</rasd:ElementName>
4574                                 <rasd:HostResource
4575                                     xmlns:vcloud="http://www.vmware.com/vcloud/v1.5"
4576                                     vcloud:capacity="{}"
4577                                     vcloud:busSubType="{}"
4578                                     vcloud:busType="{}"></rasd:HostResource>
4579                                 <rasd:InstanceID>{}</rasd:InstanceID>
4580                                 <rasd:ResourceType>17</rasd:ResourceType>
4581                             </Item>""".format(disk_size_mb, bus_subtype, bus_type, instance_id)
4582
4583             new_data = response.content
4584             #Add new item at the bottom
4585             new_data = new_data.replace('</Item>\n</RasdItemsList>', '</Item>\n{}\n</RasdItemsList>'.format(new_item))
4586
4587             # Send PUT request to modify virtual hardware section with new disk
4588             headers['Content-Type'] = 'application/vnd.vmware.vcloud.rasdItemsList+xml; charset=ISO-8859-1'
4589
4590             response = self.perform_request(req_type='PUT',
4591                                             url=disk_href,
4592                                             data=new_data,
4593                                             headers=headers)
4594
4595             if response.status_code == 403:
4596                 add_headers = {'Content-Type': headers['Content-Type']}
4597                 response = self.retry_rest('PUT', disk_href, add_headers, new_data)
4598
4599             if response.status_code != 202:
4600                 self.logger.error("PUT REST API call {} failed. Return status code {}. Response Content:{}"
4601                                   .format(disk_href, response.status_code, response.content))
4602             else:
4603                 add_disk_task = self.get_task_from_response(response.content)
4604                 result = self.client.get_task_monitor().wait_for_success(task=add_disk_task)
4605                 if result.get('status') == 'success':
4606                     status = True
4607                 else:
4608                     self.logger.error("Add new disk REST task failed to add {} MB disk".format(disk_size_mb))
4609
4610         except Exception as exp:
4611             self.logger.error("Error occurred calling rest api for creating new disk {}".format(exp))
4612
4613         return status
4614
4615
4616     def add_existing_disk(self, catalogs=None, image_id=None, size=None, template_name=None, vapp_uuid=None):
4617         """
4618             Method to add existing disk to vm
4619             Args :
4620                 catalogs - List of VDC catalogs
4621                 image_id - Catalog ID
4622                 template_name - Name of template in catalog
4623                 vapp_uuid - UUID of vApp
4624             Returns:
4625                 None
4626         """
4627         disk_info = None
4628         vcenter_conect, content = self.get_vcenter_content()
4629         #find moref-id of vm in image
4630         catalog_vm_info = self.get_vapp_template_details(catalogs=catalogs,
4631                                                          image_id=image_id,
4632                                                         )
4633
4634         if catalog_vm_info and "vm_vcenter_info" in catalog_vm_info:
4635             if "vm_moref_id" in catalog_vm_info["vm_vcenter_info"]:
4636                 catalog_vm_moref_id = catalog_vm_info["vm_vcenter_info"].get("vm_moref_id", None)
4637                 if catalog_vm_moref_id:
4638                     self.logger.info("Moref_id of VM in catalog : {}" .format(catalog_vm_moref_id))
4639                     host, catalog_vm_obj = self.get_vm_obj(content, catalog_vm_moref_id)
4640                     if catalog_vm_obj:
4641                         #find existing disk
4642                         disk_info = self.find_disk(catalog_vm_obj)
4643                     else:
4644                         exp_msg = "No VM with image id {} found".format(image_id)
4645                         self.rollback_newvm(vapp_uuid, exp_msg, exp_type="NotFound")
4646         else:
4647             exp_msg = "No Image found with image ID {} ".format(image_id)
4648             self.rollback_newvm(vapp_uuid, exp_msg, exp_type="NotFound")
4649
4650         if disk_info:
4651             self.logger.info("Existing disk_info : {}".format(disk_info))
4652             #get VM
4653             vm_moref_id = self.get_vm_moref_id(vapp_uuid)
4654             host, vm_obj = self.get_vm_obj(content, vm_moref_id)
4655             if vm_obj:
4656                 status = self.add_disk(vcenter_conect=vcenter_conect,
4657                                        vm=vm_obj,
4658                                        disk_info=disk_info,
4659                                        size=size,
4660                                        vapp_uuid=vapp_uuid
4661                                        )
4662             if status:
4663                 self.logger.info("Disk from image id {} added to {}".format(image_id,
4664                                                                             vm_obj.config.name)
4665                                  )
4666         else:
4667             msg = "No disk found with image id {} to add in VM {}".format(
4668                                                             image_id,
4669                                                             vm_obj.config.name)
4670             self.rollback_newvm(vapp_uuid, msg, exp_type="NotFound")
4671
4672
4673     def find_disk(self, vm_obj):
4674         """
4675          Method to find details of existing disk in VM
4676             Args :
4677                 vm_obj - vCenter object of VM
4678                 image_id - Catalog ID
4679             Returns:
4680                 disk_info : dict of disk details
4681         """
4682         disk_info = {}
4683         if vm_obj:
4684             try:
4685                 devices = vm_obj.config.hardware.device
4686                 for device in devices:
4687                     if type(device) is vim.vm.device.VirtualDisk:
4688                         if isinstance(device.backing,vim.vm.device.VirtualDisk.FlatVer2BackingInfo) and hasattr(device.backing, 'fileName'):
4689                             disk_info["full_path"] = device.backing.fileName
4690                             disk_info["datastore"] = device.backing.datastore
4691                             disk_info["capacityKB"] = device.capacityInKB
4692                             break
4693             except Exception as exp:
4694                 self.logger.error("find_disk() : exception occurred while "\
4695                                   "getting existing disk details :{}".format(exp))
4696         return disk_info
4697
4698
4699     def add_disk(self, vcenter_conect=None, vm=None, size=None, vapp_uuid=None, disk_info={}):
4700         """
4701          Method to add existing disk in VM
4702             Args :
4703                 vcenter_conect - vCenter content object
4704                 vm - vCenter vm object
4705                 disk_info : dict of disk details
4706             Returns:
4707                 status : status of add disk task
4708         """
4709         datastore = disk_info["datastore"] if "datastore" in disk_info else None
4710         fullpath = disk_info["full_path"] if "full_path" in disk_info else None
4711         capacityKB = disk_info["capacityKB"] if "capacityKB" in disk_info else None
4712         if size is not None:
4713             #Convert size from GB to KB
4714             sizeKB = int(size) * 1024 * 1024
4715             #compare size of existing disk and user given size.Assign whicherver is greater
4716             self.logger.info("Add Existing disk : sizeKB {} , capacityKB {}".format(
4717                                                                     sizeKB, capacityKB))
4718             if sizeKB > capacityKB:
4719                 capacityKB = sizeKB
4720
4721         if datastore and fullpath and capacityKB:
4722             try:
4723                 spec = vim.vm.ConfigSpec()
4724                 # get all disks on a VM, set unit_number to the next available
4725                 unit_number = 0
4726                 for dev in vm.config.hardware.device:
4727                     if hasattr(dev.backing, 'fileName'):
4728                         unit_number = int(dev.unitNumber) + 1
4729                         # unit_number 7 reserved for scsi controller
4730                         if unit_number == 7:
4731                             unit_number += 1
4732                     if isinstance(dev, vim.vm.device.VirtualDisk):
4733                         #vim.vm.device.VirtualSCSIController
4734                         controller_key = dev.controllerKey
4735
4736                 self.logger.info("Add Existing disk : unit number {} , controller key {}".format(
4737                                                                     unit_number, controller_key))
4738                 # add disk here
4739                 dev_changes = []
4740                 disk_spec = vim.vm.device.VirtualDeviceSpec()
4741                 disk_spec.operation = vim.vm.device.VirtualDeviceSpec.Operation.add
4742                 disk_spec.device = vim.vm.device.VirtualDisk()
4743                 disk_spec.device.backing = \
4744                     vim.vm.device.VirtualDisk.FlatVer2BackingInfo()
4745                 disk_spec.device.backing.thinProvisioned = True
4746                 disk_spec.device.backing.diskMode = 'persistent'
4747                 disk_spec.device.backing.datastore  = datastore
4748                 disk_spec.device.backing.fileName  = fullpath
4749
4750                 disk_spec.device.unitNumber = unit_number
4751                 disk_spec.device.capacityInKB = capacityKB
4752                 disk_spec.device.controllerKey = controller_key
4753                 dev_changes.append(disk_spec)
4754                 spec.deviceChange = dev_changes
4755                 task = vm.ReconfigVM_Task(spec=spec)
4756                 status = self.wait_for_vcenter_task(task, vcenter_conect)
4757                 return status
4758             except Exception as exp:
4759                 exp_msg = "add_disk() : exception {} occurred while adding disk "\
4760                           "{} to vm {}".format(exp,
4761                                                fullpath,
4762                                                vm.config.name)
4763                 self.rollback_newvm(vapp_uuid, exp_msg)
4764         else:
4765             msg = "add_disk() : Can not add disk to VM with disk info {} ".format(disk_info)
4766             self.rollback_newvm(vapp_uuid, msg)
4767
4768
4769     def get_vcenter_content(self):
4770         """
4771          Get the vsphere content object
4772         """
4773         try:
4774             vm_vcenter_info = self.get_vm_vcenter_info()
4775         except Exception as exp:
4776             self.logger.error("Error occurred while getting vCenter infromationn"\
4777                              " for VM : {}".format(exp))
4778             raise vimconn.vimconnException(message=exp)
4779
4780         context = None
4781         if hasattr(ssl, '_create_unverified_context'):
4782             context = ssl._create_unverified_context()
4783
4784         vcenter_conect = SmartConnect(
4785                     host=vm_vcenter_info["vm_vcenter_ip"],
4786                     user=vm_vcenter_info["vm_vcenter_user"],
4787                     pwd=vm_vcenter_info["vm_vcenter_password"],
4788                     port=int(vm_vcenter_info["vm_vcenter_port"]),
4789                     sslContext=context
4790                 )
4791         atexit.register(Disconnect, vcenter_conect)
4792         content = vcenter_conect.RetrieveContent()
4793         return vcenter_conect, content
4794
4795
4796     def get_vm_moref_id(self, vapp_uuid):
4797         """
4798         Get the moref_id of given VM
4799         """
4800         try:
4801             if vapp_uuid:
4802                 vm_details = self.get_vapp_details_rest(vapp_uuid, need_admin_access=True)
4803                 if vm_details and "vm_vcenter_info" in vm_details:
4804                     vm_moref_id = vm_details["vm_vcenter_info"].get("vm_moref_id", None)
4805             return vm_moref_id
4806
4807         except Exception as exp:
4808             self.logger.error("Error occurred while getting VM moref ID "\
4809                              " for VM : {}".format(exp))
4810             return None
4811
4812
4813     def get_vapp_template_details(self, catalogs=None, image_id=None , template_name=None):
4814         """
4815             Method to get vApp template details
4816                 Args :
4817                     catalogs - list of VDC catalogs
4818                     image_id - Catalog ID to find
4819                     template_name : template name in catalog
4820                 Returns:
4821                     parsed_respond : dict of vApp tempalte details
4822         """
4823         parsed_response = {}
4824
4825         vca = self.connect_as_admin()
4826         if not vca:
4827             raise vimconn.vimconnConnectionException("Failed to connect vCD")
4828
4829         try:
4830             org, vdc = self.get_vdc_details()
4831             catalog = self.get_catalog_obj(image_id, catalogs)
4832             if catalog:
4833                 items = org.get_catalog_item(catalog.get('name'), catalog.get('name'))
4834                 catalog_items = [items.attrib]
4835
4836                 if len(catalog_items) == 1:
4837                     headers = {'Accept':'application/*+xml;version=' + API_VERSION,
4838                            'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
4839
4840                     response = self.perform_request(req_type='GET',
4841                                                     url=catalog_items[0].get('href'),
4842                                                     headers=headers)
4843                     catalogItem = XmlElementTree.fromstring(response.content)
4844                     entity = [child for child in catalogItem if child.get("type") == "application/vnd.vmware.vcloud.vAppTemplate+xml"][0]
4845                     vapp_tempalte_href = entity.get("href")
4846                     #get vapp details and parse moref id
4847
4848                     namespaces = {"vssd":"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData" ,
4849                                   'ovf': 'http://schemas.dmtf.org/ovf/envelope/1',
4850                                   'vmw': 'http://www.vmware.com/schema/ovf',
4851                                   'vm': 'http://www.vmware.com/vcloud/v1.5',
4852                                   'rasd':"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData",
4853                                   'vmext':"http://www.vmware.com/vcloud/extension/v1.5",
4854                                   'xmlns':"http://www.vmware.com/vcloud/v1.5"
4855                                 }
4856
4857                     if vca._session:
4858                         response = self.perform_request(req_type='GET',
4859                                                     url=vapp_tempalte_href,
4860                                                     headers=headers)
4861
4862                         if response.status_code != requests.codes.ok:
4863                             self.logger.debug("REST API call {} failed. Return status code {}".format(
4864                                                 vapp_tempalte_href, response.status_code))
4865
4866                         else:
4867                             xmlroot_respond = XmlElementTree.fromstring(response.content)
4868                             children_section = xmlroot_respond.find('vm:Children/', namespaces)
4869                             if children_section is not None:
4870                                 vCloud_extension_section = children_section.find('xmlns:VCloudExtension', namespaces)
4871                             if vCloud_extension_section is not None:
4872                                 vm_vcenter_info = {}
4873                                 vim_info = vCloud_extension_section.find('vmext:VmVimInfo', namespaces)
4874                                 vmext = vim_info.find('vmext:VmVimObjectRef', namespaces)
4875                                 if vmext is not None:
4876                                     vm_vcenter_info["vm_moref_id"] = vmext.find('vmext:MoRef', namespaces).text
4877                                 parsed_response["vm_vcenter_info"]= vm_vcenter_info
4878
4879         except Exception as exp :
4880             self.logger.info("Error occurred calling rest api for getting vApp details {}".format(exp))
4881
4882         return parsed_response
4883
4884
4885     def rollback_newvm(self, vapp_uuid, msg , exp_type="Genric"):
4886         """
4887             Method to delete vApp
4888                 Args :
4889                     vapp_uuid - vApp UUID
4890                     msg - Error message to be logged
4891                     exp_type : Exception type
4892                 Returns:
4893                     None
4894         """
4895         if vapp_uuid:
4896             status = self.delete_vminstance(vapp_uuid)
4897         else:
4898             msg = "No vApp ID"
4899         self.logger.error(msg)
4900         if exp_type == "Genric":
4901             raise vimconn.vimconnException(msg)
4902         elif exp_type == "NotFound":
4903             raise vimconn.vimconnNotFoundException(message=msg)
4904
4905     def add_sriov(self, vapp_uuid, sriov_nets, vmname_andid):
4906         """
4907             Method to attach SRIOV adapters to VM
4908
4909              Args:
4910                 vapp_uuid - uuid of vApp/VM
4911                 sriov_nets - SRIOV devices infromation as specified in VNFD (flavor)
4912                 vmname_andid - vmname
4913
4914             Returns:
4915                 The status of add SRIOV adapter task , vm object and
4916                 vcenter_conect object
4917         """
4918         vm_obj = None
4919         vcenter_conect, content = self.get_vcenter_content()
4920         vm_moref_id = self.get_vm_moref_id(vapp_uuid)
4921
4922         if vm_moref_id:
4923             try:
4924                 no_of_sriov_devices = len(sriov_nets)
4925                 if no_of_sriov_devices > 0:
4926                     #Get VM and its host
4927                     host_obj, vm_obj = self.get_vm_obj(content, vm_moref_id)
4928                     self.logger.info("VM {} is currently on host {}".format(vm_obj, host_obj))
4929                     if host_obj and vm_obj:
4930                         #get SRIOV devies from host on which vapp is currently installed
4931                         avilable_sriov_devices = self.get_sriov_devices(host_obj,
4932                                                                 no_of_sriov_devices,
4933                                                                 )
4934
4935                         if len(avilable_sriov_devices) == 0:
4936                             #find other hosts with active pci devices
4937                             new_host_obj , avilable_sriov_devices = self.get_host_and_sriov_devices(
4938                                                                 content,
4939                                                                 no_of_sriov_devices,
4940                                                                 )
4941
4942                             if new_host_obj is not None and len(avilable_sriov_devices)> 0:
4943                                 #Migrate vm to the host where SRIOV devices are available
4944                                 self.logger.info("Relocate VM {} on new host {}".format(vm_obj,
4945                                                                                     new_host_obj))
4946                                 task = self.relocate_vm(new_host_obj, vm_obj)
4947                                 if task is not None:
4948                                     result = self.wait_for_vcenter_task(task, vcenter_conect)
4949                                     self.logger.info("Migrate VM status: {}".format(result))
4950                                     host_obj = new_host_obj
4951                                 else:
4952                                     self.logger.info("Fail to migrate VM : {}".format(result))
4953                                     raise vimconn.vimconnNotFoundException(
4954                                     "Fail to migrate VM : {} to host {}".format(
4955                                                     vmname_andid,
4956                                                     new_host_obj)
4957                                         )
4958
4959                         if host_obj is not None and avilable_sriov_devices is not None and len(avilable_sriov_devices)> 0:
4960                             #Add SRIOV devices one by one
4961                             for sriov_net in sriov_nets:
4962                                 network_name = sriov_net.get('net_id')
4963                                 dvs_portgr_name = self.create_dvPort_group(network_name)
4964                                 if sriov_net.get('type') == "VF" or sriov_net.get('type') == "SR-IOV":
4965                                     #add vlan ID ,Modify portgroup for vlan ID
4966                                     self.configure_vlanID(content, vcenter_conect, network_name)
4967
4968                                 task = self.add_sriov_to_vm(content,
4969                                                             vm_obj,
4970                                                             host_obj,
4971                                                             network_name,
4972                                                             avilable_sriov_devices[0]
4973                                                             )
4974                                 if task:
4975                                     status= self.wait_for_vcenter_task(task, vcenter_conect)
4976                                     if status:
4977                                         self.logger.info("Added SRIOV {} to VM {}".format(
4978                                                                         no_of_sriov_devices,
4979                                                                         str(vm_obj)))
4980                                 else:
4981                                     self.logger.error("Fail to add SRIOV {} to VM {}".format(
4982                                                                         no_of_sriov_devices,
4983                                                                         str(vm_obj)))
4984                                     raise vimconn.vimconnUnexpectedResponse(
4985                                     "Fail to add SRIOV adapter in VM ".format(str(vm_obj))
4986                                         )
4987                             return True, vm_obj, vcenter_conect
4988                         else:
4989                             self.logger.error("Currently there is no host with"\
4990                                               " {} number of avaialble SRIOV "\
4991                                               "VFs required for VM {}".format(
4992                                                                 no_of_sriov_devices,
4993                                                                 vmname_andid)
4994                                               )
4995                             raise vimconn.vimconnNotFoundException(
4996                                     "Currently there is no host with {} "\
4997                                     "number of avaialble SRIOV devices required for VM {}".format(
4998                                                                             no_of_sriov_devices,
4999                                                                             vmname_andid))
5000                 else:
5001                     self.logger.debug("No infromation about SRIOV devices {} ",sriov_nets)
5002
5003             except vmodl.MethodFault as error:
5004                 self.logger.error("Error occurred while adding SRIOV {} ",error)
5005         return None, vm_obj, vcenter_conect
5006
5007
5008     def get_sriov_devices(self,host, no_of_vfs):
5009         """
5010             Method to get the details of SRIOV devices on given host
5011              Args:
5012                 host - vSphere host object
5013                 no_of_vfs - number of VFs needed on host
5014
5015              Returns:
5016                 array of SRIOV devices
5017         """
5018         sriovInfo=[]
5019         if host:
5020             for device in host.config.pciPassthruInfo:
5021                 if isinstance(device,vim.host.SriovInfo) and device.sriovActive:
5022                     if device.numVirtualFunction >= no_of_vfs:
5023                         sriovInfo.append(device)
5024                         break
5025         return sriovInfo
5026
5027
5028     def get_host_and_sriov_devices(self, content, no_of_vfs):
5029         """
5030          Method to get the details of SRIOV devices infromation on all hosts
5031
5032             Args:
5033                 content - vSphere host object
5034                 no_of_vfs - number of pci VFs needed on host
5035
5036             Returns:
5037                  array of SRIOV devices and host object
5038         """
5039         host_obj = None
5040         sriov_device_objs = None
5041         try:
5042             if content:
5043                 container = content.viewManager.CreateContainerView(content.rootFolder,
5044                                                             [vim.HostSystem], True)
5045                 for host in container.view:
5046                     devices = self.get_sriov_devices(host, no_of_vfs)
5047                     if devices:
5048                         host_obj = host
5049                         sriov_device_objs = devices
5050                         break
5051         except Exception as exp:
5052             self.logger.error("Error {} occurred while finding SRIOV devices on host: {}".format(exp, host_obj))
5053
5054         return host_obj,sriov_device_objs
5055
5056
5057     def add_sriov_to_vm(self,content, vm_obj, host_obj, network_name, sriov_device):
5058         """
5059          Method to add SRIOV adapter to vm
5060
5061             Args:
5062                 host_obj - vSphere host object
5063                 vm_obj - vSphere vm object
5064                 content - vCenter content object
5065                 network_name - name of distributed virtaul portgroup
5066                 sriov_device - SRIOV device info
5067
5068             Returns:
5069                  task object
5070         """
5071         devices = []
5072         vnic_label = "sriov nic"
5073         try:
5074             dvs_portgr = self.get_dvport_group(network_name)
5075             network_name = dvs_portgr.name
5076             nic = vim.vm.device.VirtualDeviceSpec()
5077             # VM device
5078             nic.operation = vim.vm.device.VirtualDeviceSpec.Operation.add
5079             nic.device = vim.vm.device.VirtualSriovEthernetCard()
5080             nic.device.addressType = 'assigned'
5081             #nic.device.key = 13016
5082             nic.device.deviceInfo = vim.Description()
5083             nic.device.deviceInfo.label = vnic_label
5084             nic.device.deviceInfo.summary = network_name
5085             nic.device.backing = vim.vm.device.VirtualEthernetCard.NetworkBackingInfo()
5086
5087             nic.device.backing.network = self.get_obj(content, [vim.Network], network_name)
5088             nic.device.backing.deviceName = network_name
5089             nic.device.backing.useAutoDetect = False
5090             nic.device.connectable = vim.vm.device.VirtualDevice.ConnectInfo()
5091             nic.device.connectable.startConnected = True
5092             nic.device.connectable.allowGuestControl = True
5093
5094             nic.device.sriovBacking = vim.vm.device.VirtualSriovEthernetCard.SriovBackingInfo()
5095             nic.device.sriovBacking.physicalFunctionBacking = vim.vm.device.VirtualPCIPassthrough.DeviceBackingInfo()
5096             nic.device.sriovBacking.physicalFunctionBacking.id = sriov_device.id
5097
5098             devices.append(nic)
5099             vmconf = vim.vm.ConfigSpec(deviceChange=devices)
5100             task = vm_obj.ReconfigVM_Task(vmconf)
5101             return task
5102         except Exception as exp:
5103             self.logger.error("Error {} occurred while adding SRIOV adapter in VM: {}".format(exp, vm_obj))
5104             return None
5105
5106
5107     def create_dvPort_group(self, network_name):
5108         """
5109          Method to create disributed virtual portgroup
5110
5111             Args:
5112                 network_name - name of network/portgroup
5113
5114             Returns:
5115                 portgroup key
5116         """
5117         try:
5118             new_network_name = [network_name, '-', str(uuid.uuid4())]
5119             network_name=''.join(new_network_name)
5120             vcenter_conect, content = self.get_vcenter_content()
5121
5122             dv_switch = self.get_obj(content, [vim.DistributedVirtualSwitch], self.dvs_name)
5123             if dv_switch:
5124                 dv_pg_spec = vim.dvs.DistributedVirtualPortgroup.ConfigSpec()
5125                 dv_pg_spec.name = network_name
5126
5127                 dv_pg_spec.type = vim.dvs.DistributedVirtualPortgroup.PortgroupType.earlyBinding
5128                 dv_pg_spec.defaultPortConfig = vim.dvs.VmwareDistributedVirtualSwitch.VmwarePortConfigPolicy()
5129                 dv_pg_spec.defaultPortConfig.securityPolicy = vim.dvs.VmwareDistributedVirtualSwitch.SecurityPolicy()
5130                 dv_pg_spec.defaultPortConfig.securityPolicy.allowPromiscuous = vim.BoolPolicy(value=False)
5131                 dv_pg_spec.defaultPortConfig.securityPolicy.forgedTransmits = vim.BoolPolicy(value=False)
5132                 dv_pg_spec.defaultPortConfig.securityPolicy.macChanges = vim.BoolPolicy(value=False)
5133
5134                 task = dv_switch.AddDVPortgroup_Task([dv_pg_spec])
5135                 self.wait_for_vcenter_task(task, vcenter_conect)
5136
5137                 dvPort_group = self.get_obj(content, [vim.dvs.DistributedVirtualPortgroup], network_name)
5138                 if dvPort_group:
5139                     self.logger.info("Created disributed virtaul port group: {}".format(dvPort_group))
5140                     return dvPort_group.key
5141             else:
5142                 self.logger.debug("No disributed virtual switch found with name {}".format(network_name))
5143
5144         except Exception as exp:
5145             self.logger.error("Error occurred while creating disributed virtaul port group {}"\
5146                              " : {}".format(network_name, exp))
5147         return None
5148
5149     def reconfig_portgroup(self, content, dvPort_group_name , config_info={}):
5150         """
5151          Method to reconfigure disributed virtual portgroup
5152
5153             Args:
5154                 dvPort_group_name - name of disributed virtual portgroup
5155                 content - vCenter content object
5156                 config_info - disributed virtual portgroup configuration
5157
5158             Returns:
5159                 task object
5160         """
5161         try:
5162             dvPort_group = self.get_dvport_group(dvPort_group_name)
5163             if dvPort_group:
5164                 dv_pg_spec = vim.dvs.DistributedVirtualPortgroup.ConfigSpec()
5165                 dv_pg_spec.configVersion = dvPort_group.config.configVersion
5166                 dv_pg_spec.defaultPortConfig = vim.dvs.VmwareDistributedVirtualSwitch.VmwarePortConfigPolicy()
5167                 if "vlanID" in config_info:
5168                     dv_pg_spec.defaultPortConfig.vlan = vim.dvs.VmwareDistributedVirtualSwitch.VlanIdSpec()
5169                     dv_pg_spec.defaultPortConfig.vlan.vlanId = config_info.get('vlanID')
5170
5171                 task = dvPort_group.ReconfigureDVPortgroup_Task(spec=dv_pg_spec)
5172                 return task
5173             else:
5174                 return None
5175         except Exception as exp:
5176             self.logger.error("Error occurred while reconfiguraing disributed virtaul port group {}"\
5177                              " : {}".format(dvPort_group_name, exp))
5178             return None
5179
5180
5181     def destroy_dvport_group(self , dvPort_group_name):
5182         """
5183          Method to destroy disributed virtual portgroup
5184
5185             Args:
5186                 network_name - name of network/portgroup
5187
5188             Returns:
5189                 True if portgroup successfully got deleted else false
5190         """
5191         vcenter_conect, content = self.get_vcenter_content()
5192         try:
5193             status = None
5194             dvPort_group = self.get_dvport_group(dvPort_group_name)
5195             if dvPort_group:
5196                 task = dvPort_group.Destroy_Task()
5197                 status = self.wait_for_vcenter_task(task, vcenter_conect)
5198             return status
5199         except vmodl.MethodFault as exp:
5200             self.logger.error("Caught vmodl fault {} while deleting disributed virtaul port group {}".format(
5201                                                                     exp, dvPort_group_name))
5202             return None
5203
5204
5205     def get_dvport_group(self, dvPort_group_name):
5206         """
5207         Method to get disributed virtual portgroup
5208
5209             Args:
5210                 network_name - name of network/portgroup
5211
5212             Returns:
5213                 portgroup object
5214         """
5215         vcenter_conect, content = self.get_vcenter_content()
5216         dvPort_group = None
5217         try:
5218             container = content.viewManager.CreateContainerView(content.rootFolder, [vim.dvs.DistributedVirtualPortgroup], True)
5219             for item in container.view:
5220                 if item.key == dvPort_group_name:
5221                     dvPort_group = item
5222                     break
5223             return dvPort_group
5224         except vmodl.MethodFault as exp:
5225             self.logger.error("Caught vmodl fault {} for disributed virtaul port group {}".format(
5226                                                                             exp, dvPort_group_name))
5227             return None
5228
5229     def get_vlanID_from_dvs_portgr(self, dvPort_group_name):
5230         """
5231          Method to get disributed virtual portgroup vlanID
5232
5233             Args:
5234                 network_name - name of network/portgroup
5235
5236             Returns:
5237                 vlan ID
5238         """
5239         vlanId = None
5240         try:
5241             dvPort_group = self.get_dvport_group(dvPort_group_name)
5242             if dvPort_group:
5243                 vlanId = dvPort_group.config.defaultPortConfig.vlan.vlanId
5244         except vmodl.MethodFault as exp:
5245             self.logger.error("Caught vmodl fault {} for disributed virtaul port group {}".format(
5246                                                                             exp, dvPort_group_name))
5247         return vlanId
5248
5249
5250     def configure_vlanID(self, content, vcenter_conect, dvPort_group_name):
5251         """
5252          Method to configure vlanID in disributed virtual portgroup vlanID
5253
5254             Args:
5255                 network_name - name of network/portgroup
5256
5257             Returns:
5258                 None
5259         """
5260         vlanID = self.get_vlanID_from_dvs_portgr(dvPort_group_name)
5261         if vlanID == 0:
5262             #configure vlanID
5263             vlanID = self.genrate_vlanID(dvPort_group_name)
5264             config = {"vlanID":vlanID}
5265             task = self.reconfig_portgroup(content, dvPort_group_name,
5266                                     config_info=config)
5267             if task:
5268                 status= self.wait_for_vcenter_task(task, vcenter_conect)
5269                 if status:
5270                     self.logger.info("Reconfigured Port group {} for vlan ID {}".format(
5271                                                         dvPort_group_name,vlanID))
5272             else:
5273                 self.logger.error("Fail reconfigure portgroup {} for vlanID{}".format(
5274                                         dvPort_group_name, vlanID))
5275
5276
5277     def genrate_vlanID(self, network_name):
5278         """
5279          Method to get unused vlanID
5280             Args:
5281                 network_name - name of network/portgroup
5282             Returns:
5283                 vlanID
5284         """
5285         vlan_id = None
5286         used_ids = []
5287         if self.config.get('vlanID_range') == None:
5288             raise vimconn.vimconnConflictException("You must provide a 'vlanID_range' "\
5289                         "at config value before creating sriov network with vlan tag")
5290         if "used_vlanIDs" not in self.persistent_info:
5291                 self.persistent_info["used_vlanIDs"] = {}
5292         else:
5293             used_ids = self.persistent_info["used_vlanIDs"].values()
5294             #For python3
5295             #used_ids = list(self.persistent_info["used_vlanIDs"].values())
5296
5297         for vlanID_range in self.config.get('vlanID_range'):
5298             start_vlanid , end_vlanid = vlanID_range.split("-")
5299             if start_vlanid > end_vlanid:
5300                 raise vimconn.vimconnConflictException("Invalid vlan ID range {}".format(
5301                                                                         vlanID_range))
5302
5303             for id in xrange(int(start_vlanid), int(end_vlanid) + 1):
5304             #For python3
5305             #for id in range(int(start_vlanid), int(end_vlanid) + 1):
5306                 if id not in used_ids:
5307                     vlan_id = id
5308                     self.persistent_info["used_vlanIDs"][network_name] = vlan_id
5309                     return vlan_id
5310         if vlan_id is None:
5311             raise vimconn.vimconnConflictException("All Vlan IDs are in use")
5312
5313
5314     def get_obj(self, content, vimtype, name):
5315         """
5316          Get the vsphere object associated with a given text name
5317         """
5318         obj = None
5319         container = content.viewManager.CreateContainerView(content.rootFolder, vimtype, True)
5320         for item in container.view:
5321             if item.name == name:
5322                 obj = item
5323                 break
5324         return obj
5325
5326
5327     def insert_media_to_vm(self, vapp, image_id):
5328         """
5329         Method to insert media CD-ROM (ISO image) from catalog to vm.
5330         vapp - vapp object to get vm id
5331         Image_id - image id for cdrom to be inerted to vm
5332         """
5333         # create connection object
5334         vca = self.connect()
5335         try:
5336             # fetching catalog details
5337             rest_url = "{}/api/catalog/{}".format(self.url, image_id)
5338             if vca._session:
5339                 headers = {'Accept':'application/*+xml;version=' + API_VERSION,
5340                            'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
5341                 response = self.perform_request(req_type='GET',
5342                                                 url=rest_url,
5343                                                 headers=headers)
5344
5345             if response.status_code != 200:
5346                 self.logger.error("REST call {} failed reason : {}"\
5347                              "status code : {}".format(url_rest_call,
5348                                                     response.content,
5349                                                response.status_code))
5350                 raise vimconn.vimconnException("insert_media_to_vm(): Failed to get "\
5351                                                                     "catalog details")
5352             # searching iso name and id
5353             iso_name,media_id = self.get_media_details(vca, response.content)
5354
5355             if iso_name and media_id:
5356                 data ="""<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
5357                      <ns6:MediaInsertOrEjectParams
5358                      xmlns="http://www.vmware.com/vcloud/versions" xmlns:ns2="http://schemas.dmtf.org/ovf/envelope/1" xmlns:ns3="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData" xmlns:ns4="http://schemas.dmtf.org/wbem/wscim/1/common" xmlns:ns5="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData" xmlns:ns6="http://www.vmware.com/vcloud/v1.5" xmlns:ns7="http://www.vmware.com/schema/ovf" xmlns:ns8="http://schemas.dmtf.org/ovf/environment/1" xmlns:ns9="http://www.vmware.com/vcloud/extension/v1.5">
5359                      <ns6:Media
5360                         type="application/vnd.vmware.vcloud.media+xml"
5361                         name="{}.iso"
5362                         id="urn:vcloud:media:{}"
5363                         href="https://{}/api/media/{}"/>
5364                      </ns6:MediaInsertOrEjectParams>""".format(iso_name, media_id,
5365                                                                 self.url,media_id)
5366
5367                 for vms in vapp.get_all_vms():
5368                     vm_id = vms.get('id').split(':')[-1]
5369
5370                     headers['Content-Type'] = 'application/vnd.vmware.vcloud.mediaInsertOrEjectParams+xml'
5371                     rest_url = "{}/api/vApp/vm-{}/media/action/insertMedia".format(self.url,vm_id)
5372
5373                     response = self.perform_request(req_type='POST',
5374                                                        url=rest_url,
5375                                                           data=data,
5376                                                     headers=headers)
5377
5378                     if response.status_code != 202:
5379                         self.logger.error("Failed to insert CD-ROM to vm")
5380                         raise vimconn.vimconnException("insert_media_to_vm() : Failed to insert"\
5381                                                                                     "ISO image to vm")
5382                     else:
5383                         task = self.get_task_from_response(response.content)
5384                         result = self.client.get_task_monitor().wait_for_success(task=task)
5385                         if result.get('status') == 'success':
5386                             self.logger.info("insert_media_to_vm(): Sucessfully inserted media ISO"\
5387                                                                     " image to vm {}".format(vm_id))
5388
5389         except Exception as exp:
5390             self.logger.error("insert_media_to_vm() : exception occurred "\
5391                                             "while inserting media CD-ROM")
5392             raise vimconn.vimconnException(message=exp)
5393
5394
5395     def get_media_details(self, vca, content):
5396         """
5397         Method to get catalog item details
5398         vca - connection object
5399         content - Catalog details
5400         Return - Media name, media id
5401         """
5402         cataloghref_list = []
5403         try:
5404             if content:
5405                 vm_list_xmlroot = XmlElementTree.fromstring(content)
5406                 for child in vm_list_xmlroot.iter():
5407                     if 'CatalogItem' in child.tag:
5408                         cataloghref_list.append(child.attrib.get('href'))
5409                 if cataloghref_list is not None:
5410                     for href in cataloghref_list:
5411                         if href:
5412                             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
5413                            'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
5414                             response = self.perform_request(req_type='GET',
5415                                                                   url=href,
5416                                                            headers=headers)
5417                             if response.status_code != 200:
5418                                 self.logger.error("REST call {} failed reason : {}"\
5419                                              "status code : {}".format(href,
5420                                                            response.content,
5421                                                       response.status_code))
5422                                 raise vimconn.vimconnException("get_media_details : Failed to get "\
5423                                                                          "catalogitem details")
5424                             list_xmlroot = XmlElementTree.fromstring(response.content)
5425                             for child in list_xmlroot.iter():
5426                                 if 'Entity' in child.tag:
5427                                     if 'media' in child.attrib.get('href'):
5428                                         name = child.attrib.get('name')
5429                                         media_id = child.attrib.get('href').split('/').pop()
5430                                         return name,media_id
5431                             else:
5432                                 self.logger.debug("Media name and id not found")
5433                                 return False,False
5434         except Exception as exp:
5435             self.logger.error("get_media_details : exception occurred "\
5436                                                "getting media details")
5437             raise vimconn.vimconnException(message=exp)
5438
5439
5440     def retry_rest(self, method, url, add_headers=None, data=None):
5441         """ Method to get Token & retry respective REST request
5442             Args:
5443                 api - REST API - Can be one of 'GET' or 'PUT' or 'POST'
5444                 url - request url to be used
5445                 add_headers - Additional headers (optional)
5446                 data - Request payload data to be passed in request
5447             Returns:
5448                 response - Response of request
5449         """
5450         response = None
5451
5452         #Get token
5453         self.get_token()
5454
5455         if self.client._session:
5456                 headers = {'Accept':'application/*+xml;version=' + API_VERSION,
5457                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
5458
5459         if add_headers:
5460             headers.update(add_headers)
5461
5462         if method == 'GET':
5463             response = self.perform_request(req_type='GET',
5464                                             url=url,
5465                                             headers=headers)
5466         elif method == 'PUT':
5467             response = self.perform_request(req_type='PUT',
5468                                             url=url,
5469                                             headers=headers,
5470                                             data=data)
5471         elif method == 'POST':
5472             response = self.perform_request(req_type='POST',
5473                                             url=url,
5474                                             headers=headers,
5475                                             data=data)
5476         elif method == 'DELETE':
5477             response = self.perform_request(req_type='DELETE',
5478                                             url=url,
5479                                             headers=headers)
5480         return response
5481
5482
5483     def get_token(self):
5484         """ Generate a new token if expired
5485
5486             Returns:
5487                 The return client object that letter can be used to connect to vCloud director as admin for VDC
5488         """
5489         try:
5490             self.logger.debug("Generate token for vca {} as {} to datacenter {}.".format(self.org_name,
5491                                                                                       self.user,
5492                                                                                       self.org_name))
5493             host = self.url
5494             client = Client(host, verify_ssl_certs=False)
5495             client.set_credentials(BasicLoginCredentials(self.user, self.org_name, self.passwd))
5496             # connection object
5497             self.client = client
5498
5499         except:
5500             raise vimconn.vimconnConnectionException("Can't connect to a vCloud director org: "
5501                                                      "{} as user: {}".format(self.org_name, self.user))
5502
5503         if not client:
5504             raise vimconn.vimconnConnectionException("Failed while reconnecting vCD")
5505
5506
5507     def get_vdc_details(self):
5508         """ Get VDC details using pyVcloud Lib
5509
5510             Returns org and vdc object
5511         """
5512         org = Org(self.client, resource=self.client.get_org())
5513         vdc = org.get_vdc(self.tenant_name)
5514
5515         #Retry once, if failed by refreshing token
5516         if vdc is None:
5517             self.get_token()
5518             vdc = org.get_vdc(self.tenant_name)
5519
5520         return org, vdc
5521
5522
5523     def perform_request(self, req_type, url, headers=None, data=None):
5524         """Perform the POST/PUT/GET/DELETE request."""
5525
5526         #Log REST request details
5527         self.log_request(req_type, url=url, headers=headers, data=data)
5528         # perform request and return its result
5529         if req_type == 'GET':
5530             response = requests.get(url=url,
5531                                 headers=headers,
5532                                 verify=False)
5533         elif req_type == 'PUT':
5534             response = requests.put(url=url,
5535                                 headers=headers,
5536                                 data=data,
5537                                 verify=False)
5538         elif req_type == 'POST':
5539             response = requests.post(url=url,
5540                                  headers=headers,
5541                                  data=data,
5542                                  verify=False)
5543         elif req_type == 'DELETE':
5544             response = requests.delete(url=url,
5545                                  headers=headers,
5546                                  verify=False)
5547         #Log the REST response
5548         self.log_response(response)
5549
5550         return response
5551
5552
5553     def log_request(self, req_type, url=None, headers=None, data=None):
5554         """Logs REST request details"""
5555
5556         if req_type is not None:
5557             self.logger.debug("Request type: {}".format(req_type))
5558
5559         if url is not None:
5560             self.logger.debug("Request url: {}".format(url))
5561
5562         if headers is not None:
5563             for header in headers:
5564                 self.logger.debug("Request header: {}: {}".format(header, headers[header]))
5565
5566         if data is not None:
5567             self.logger.debug("Request data: {}".format(data))
5568
5569
5570     def log_response(self, response):
5571         """Logs REST response details"""
5572
5573         self.logger.debug("Response status code: {} ".format(response.status_code))
5574
5575
5576     def get_task_from_response(self, content):
5577         """
5578         content - API response content(response.content)
5579         return task object
5580         """
5581         xmlroot = XmlElementTree.fromstring(content)
5582         if xmlroot.tag.split('}')[1] == "Task":
5583             return xmlroot
5584         else:
5585             for ele in xmlroot:
5586                 if ele.tag.split("}")[1] == "Tasks":
5587                     task = ele[0]
5588                     break
5589             return task
5590
5591
5592     def power_on_vapp(self,vapp_id, vapp_name):
5593         """
5594         vapp_id - vApp uuid
5595         vapp_name - vAapp name
5596         return - Task object
5597         """
5598         headers = {'Accept':'application/*+xml;version=' + API_VERSION,
5599                    'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
5600
5601         poweron_href = "{}/api/vApp/vapp-{}/power/action/powerOn".format(self.url,
5602                                                                           vapp_id)
5603         response = self.perform_request(req_type='POST',
5604                                        url=poweron_href,
5605                                         headers=headers)
5606
5607         if response.status_code != 202:
5608             self.logger.error("REST call {} failed reason : {}"\
5609                          "status code : {} ".format(poweron_href,
5610                                                 response.content,
5611                                            response.status_code))
5612             raise vimconn.vimconnException("power_on_vapp() : Failed to power on "\
5613                                                       "vApp {}".format(vapp_name))
5614         else:
5615             poweron_task = self.get_task_from_response(response.content)
5616             return poweron_task
5617
5618