osm_ro/vimconn_vmware.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2016-2017 VMware Inc.
   5 # This file is part of ETSI OSM
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact:  osslegalrouting@vmware.com
  22 ##
  23
  24 """
  25 vimconn_vmware implementation an Abstract class in order to interact with VMware  vCloud Director.
  26 mbayramov@vmware.com
  27 """
  28 from progressbar import Percentage, Bar, ETA, FileTransferSpeed, ProgressBar
  29
  30 import vimconn
  31 import os
  32 import traceback
  33 import itertools
  34 import requests
  35 import ssl
  36 import atexit
  37
  38 from pyVmomi import vim, vmodl
  39 from pyVim.connect import SmartConnect, Disconnect
  40
  41 from xml.etree import ElementTree as XmlElementTree
  42 from lxml import etree as lxmlElementTree
  43
  44 import yaml
  45 from pyvcloud.vcd.client import BasicLoginCredentials,Client,VcdTaskException
  46 from pyvcloud.vcd.vdc import VDC
  47 from pyvcloud.vcd.org import Org
  48 import re
  49 from pyvcloud.vcd.vapp import VApp
  50 from xml.sax.saxutils import escape
  51 import logging
  52 import json
  53 import time
  54 import uuid
  55 import httplib
  56 #For python3
  57 #import http.client
  58 import hashlib
  59 import socket
  60 import struct
  61 import netaddr
  62 import random
  63
  64 # global variable for vcd connector type
  65 STANDALONE = 'standalone'
  66
  67 # key for flavor dicts
  68 FLAVOR_RAM_KEY = 'ram'
  69 FLAVOR_VCPUS_KEY = 'vcpus'
  70 FLAVOR_DISK_KEY = 'disk'
  71 DEFAULT_IP_PROFILE = {'dhcp_count':50,
  72                       'dhcp_enabled':True,
  73                       'ip_version':"IPv4"
  74                       }
  75 # global variable for wait time
  76 INTERVAL_TIME = 5
  77 MAX_WAIT_TIME = 1800
  78
  79 API_VERSION = '5.9'
  80
  81 __author__ = "Mustafa Bayramov, Arpita Kate, Sachin Bhangare, Prakash Kasar"
  82 __date__ = "$09-Mar-2018 11:09:29$"
  83 __version__ = '0.2'
  84
  85 #     -1: "Could not be created",
  86 #     0: "Unresolved",
  87 #     1: "Resolved",
  88 #     2: "Deployed",
  89 #     3: "Suspended",
  90 #     4: "Powered on",
  91 #     5: "Waiting for user input",
  92 #     6: "Unknown state",
  93 #     7: "Unrecognized state",
  94 #     8: "Powered off",
  95 #     9: "Inconsistent state",
  96 #     10: "Children do not all have the same status",
  97 #     11: "Upload initiated, OVF descriptor pending",
  98 #     12: "Upload initiated, copying contents",
  99 #     13: "Upload initiated , disk contents pending",
 100 #     14: "Upload has been quarantined",
 101 #     15: "Upload quarantine period has expired"
 102
 103 # mapping vCD status to MANO
 104 vcdStatusCode2manoFormat = {4: 'ACTIVE',
 105                             7: 'PAUSED',
 106                             3: 'SUSPENDED',
 107                             8: 'INACTIVE',
 108                             12: 'BUILD',
 109                             -1: 'ERROR',
 110                             14: 'DELETED'}
 111
 112 #
 113 netStatus2manoFormat = {'ACTIVE': 'ACTIVE', 'PAUSED': 'PAUSED', 'INACTIVE': 'INACTIVE', 'BUILD': 'BUILD',
 114                         'ERROR': 'ERROR', 'DELETED': 'DELETED'
 115                         }
 116
 117 class vimconnector(vimconn.vimconnector):
 118     # dict used to store flavor in memory
 119     flavorlist = {}
 120
 121     def __init__(self, uuid=None, name=None, tenant_id=None, tenant_name=None,
 122                  url=None, url_admin=None, user=None, passwd=None, log_level=None, config={}, persistent_info={}):
 123         """
 124         Constructor create vmware connector to vCloud director.
 125
 126         By default construct doesn't validate connection state. So client can create object with None arguments.
 127         If client specified username , password and host and VDC name.  Connector initialize other missing attributes.
 128
 129         a) It initialize organization UUID
 130         b) Initialize tenant_id/vdc ID.   (This information derived from tenant name)
 131
 132         Args:
 133             uuid - is organization uuid.
 134             name - is organization name that must be presented in vCloud director.
 135             tenant_id - is VDC uuid it must be presented in vCloud director
 136             tenant_name - is VDC name.
 137             url - is hostname or ip address of vCloud director
 138             url_admin - same as above.
 139             user - is user that administrator for organization. Caller must make sure that
 140                     username has right privileges.
 141
 142             password - is password for a user.
 143
 144             VMware connector also requires PVDC administrative privileges and separate account.
 145             This variables must be passed via config argument dict contains keys
 146
 147             dict['admin_username']
 148             dict['admin_password']
 149             config - Provide NSX and vCenter information
 150
 151             Returns:
 152                 Nothing.
 153         """
 154
 155         vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url,
 156                                       url_admin, user, passwd, log_level, config)
 157
 158         self.logger = logging.getLogger('openmano.vim.vmware')
 159         self.logger.setLevel(10)
 160         self.persistent_info = persistent_info
 161
 162         self.name = name
 163         self.id = uuid
 164         self.url = url
 165         self.url_admin = url_admin
 166         self.tenant_id = tenant_id
 167         self.tenant_name = tenant_name
 168         self.user = user
 169         self.passwd = passwd
 170         self.config = config
 171         self.admin_password = None
 172         self.admin_user = None
 173         self.org_name = ""
 174         self.nsx_manager = None
 175         self.nsx_user = None
 176         self.nsx_password = None
 177         self.availability_zone = None
 178
 179         # Disable warnings from self-signed certificates.
 180         requests.packages.urllib3.disable_warnings()
 181
 182         if tenant_name is not None:
 183             orgnameandtenant = tenant_name.split(":")
 184             if len(orgnameandtenant) == 2:
 185                 self.tenant_name = orgnameandtenant[1]
 186                 self.org_name = orgnameandtenant[0]
 187             else:
 188                 self.tenant_name = tenant_name
 189         if "orgname" in config:
 190             self.org_name = config['orgname']
 191
 192         if log_level:
 193             self.logger.setLevel(getattr(logging, log_level))
 194
 195         try:
 196             self.admin_user = config['admin_username']
 197             self.admin_password = config['admin_password']
 198         except KeyError:
 199             raise vimconn.vimconnException(message="Error admin username or admin password is empty.")
 200
 201         try:
 202             self.nsx_manager = config['nsx_manager']
 203             self.nsx_user = config['nsx_user']
 204             self.nsx_password = config['nsx_password']
 205         except KeyError:
 206             raise vimconn.vimconnException(message="Error: nsx manager or nsx user or nsx password is empty in Config")
 207
 208         self.vcenter_ip = config.get("vcenter_ip", None)
 209         self.vcenter_port = config.get("vcenter_port", None)
 210         self.vcenter_user = config.get("vcenter_user", None)
 211         self.vcenter_password = config.get("vcenter_password", None)
 212
 213         #Set availability zone for Affinity rules
 214         self.availability_zone = self.set_availability_zones()
 215
 216 # ############# Stub code for SRIOV #################
 217 #         try:
 218 #             self.dvs_name = config['dv_switch_name']
 219 #         except KeyError:
 220 #             raise vimconn.vimconnException(message="Error: distributed virtaul switch name is empty in Config")
 221 #
 222 #         self.vlanID_range = config.get("vlanID_range", None)
 223
 224         self.org_uuid = None
 225         self.client = None
 226
 227         if not url:
 228             raise vimconn.vimconnException('url param can not be NoneType')
 229
 230         if not self.url_admin:  # try to use normal url
 231             self.url_admin = self.url
 232
 233         logging.debug("UUID: {} name: {} tenant_id: {} tenant name {}".format(self.id, self.org_name,
 234                                                                               self.tenant_id, self.tenant_name))
 235         logging.debug("vcd url {} vcd username: {} vcd password: {}".format(self.url, self.user, self.passwd))
 236         logging.debug("vcd admin username {} vcd admin passowrd {}".format(self.admin_user, self.admin_password))
 237
 238         # initialize organization
 239         if self.user is not None and self.passwd is not None and self.url:
 240             self.init_organization()
 241
 242     def __getitem__(self, index):
 243         if index == 'name':
 244             return self.name
 245         if index == 'tenant_id':
 246             return self.tenant_id
 247         if index == 'tenant_name':
 248             return self.tenant_name
 249         elif index == 'id':
 250             return self.id
 251         elif index == 'org_name':
 252             return self.org_name
 253         elif index == 'org_uuid':
 254             return self.org_uuid
 255         elif index == 'user':
 256             return self.user
 257         elif index == 'passwd':
 258             return self.passwd
 259         elif index == 'url':
 260             return self.url
 261         elif index == 'url_admin':
 262             return self.url_admin
 263         elif index == "config":
 264             return self.config
 265         else:
 266             raise KeyError("Invalid key '%s'" % str(index))
 267
 268     def __setitem__(self, index, value):
 269         if index == 'name':
 270             self.name = value
 271         if index == 'tenant_id':
 272             self.tenant_id = value
 273         if index == 'tenant_name':
 274             self.tenant_name = value
 275         elif index == 'id':
 276             self.id = value
 277         elif index == 'org_name':
 278             self.org_name = value
 279         elif index == 'org_uuid':
 280             self.org_uuid = value
 281         elif index == 'user':
 282             self.user = value
 283         elif index == 'passwd':
 284             self.passwd = value
 285         elif index == 'url':
 286             self.url = value
 287         elif index == 'url_admin':
 288             self.url_admin = value
 289         else:
 290             raise KeyError("Invalid key '%s'" % str(index))
 291
 292     def connect_as_admin(self):
 293         """ Method connect as pvdc admin user to vCloud director.
 294             There are certain action that can be done only by provider vdc admin user.
 295             Organization creation / provider network creation etc.
 296
 297             Returns:
 298                 The return client object that latter can be used to connect to vcloud director as admin for provider vdc
 299         """
 300
 301         self.logger.debug("Logging into vCD {} as admin.".format(self.org_name))
 302
 303         try:
 304             host = self.url
 305             org = 'System'
 306             client_as_admin = Client(host, verify_ssl_certs=False)
 307             client_as_admin.set_credentials(BasicLoginCredentials(self.admin_user, org, self.admin_password))
 308         except Exception as e:
 309             raise vimconn.vimconnException(
 310                   "Can't connect to a vCloud director as: {} with exception {}".format(self.admin_user, e))
 311
 312         return client_as_admin
 313
 314     def connect(self):
 315         """ Method connect as normal user to vCloud director.
 316
 317             Returns:
 318                 The return client object that latter can be used to connect to vCloud director as admin for VDC
 319         """
 320
 321         try:
 322             self.logger.debug("Logging into vCD {} as {} to datacenter {}.".format(self.org_name,
 323                                                                                       self.user,
 324                                                                                       self.org_name))
 325             host = self.url
 326             client = Client(host, verify_ssl_certs=False)
 327             client.set_credentials(BasicLoginCredentials(self.user, self.org_name, self.passwd))
 328         except:
 329             raise vimconn.vimconnConnectionException("Can't connect to a vCloud director org: "
 330                                                      "{} as user: {}".format(self.org_name, self.user))
 331
 332         return client
 333
 334     def init_organization(self):
 335         """ Method initialize organization UUID and VDC parameters.
 336
 337             At bare minimum client must provide organization name that present in vCloud director and VDC.
 338
 339             The VDC - UUID ( tenant_id) will be initialized at the run time if client didn't call constructor.
 340             The Org - UUID will be initialized at the run time if data center present in vCloud director.
 341
 342             Returns:
 343                 The return vca object that letter can be used to connect to vcloud direct as admin
 344         """
 345         client = self.connect()
 346         if not client:
 347             raise vimconn.vimconnConnectionException("Failed to connect vCD.")
 348
 349         self.client = client
 350         try:
 351             if self.org_uuid is None:
 352                 org_list = client.get_org_list()
 353                 for org in org_list.Org:
 354                     # we set org UUID at the init phase but we can do it only when we have valid credential.
 355                     if org.get('name') == self.org_name:
 356                         self.org_uuid = org.get('href').split('/')[-1]
 357                         self.logger.debug("Setting organization UUID {}".format(self.org_uuid))
 358                         break
 359                 else:
 360                     raise vimconn.vimconnException("Vcloud director organization {} not found".format(self.org_name))
 361
 362                 # if well good we require for org details
 363                 org_details_dict = self.get_org(org_uuid=self.org_uuid)
 364
 365                 # we have two case if we want to initialize VDC ID or VDC name at run time
 366                 # tenant_name provided but no tenant id
 367                 if self.tenant_id is None and self.tenant_name is not None and 'vdcs' in org_details_dict:
 368                     vdcs_dict = org_details_dict['vdcs']
 369                     for vdc in vdcs_dict:
 370                         if vdcs_dict[vdc] == self.tenant_name:
 371                             self.tenant_id = vdc
 372                             self.logger.debug("Setting vdc uuid {} for organization UUID {}".format(self.tenant_id,
 373                                                                                                     self.org_name))
 374                             break
 375                     else:
 376                         raise vimconn.vimconnException("Tenant name indicated but not present in vcloud director.")
 377                     # case two we have tenant_id but we don't have tenant name so we find and set it.
 378                     if self.tenant_id is not None and self.tenant_name is None and 'vdcs' in org_details_dict:
 379                         vdcs_dict = org_details_dict['vdcs']
 380                         for vdc in vdcs_dict:
 381                             if vdc == self.tenant_id:
 382                                 self.tenant_name = vdcs_dict[vdc]
 383                                 self.logger.debug("Setting vdc uuid {} for organization UUID {}".format(self.tenant_id,
 384                                                                                                         self.org_name))
 385                                 break
 386                         else:
 387                             raise vimconn.vimconnException("Tenant id indicated but not present in vcloud director")
 388             self.logger.debug("Setting organization uuid {}".format(self.org_uuid))
 389         except:
 390             self.logger.debug("Failed initialize organization UUID for org {}".format(self.org_name))
 391             self.logger.debug(traceback.format_exc())
 392             self.org_uuid = None
 393
 394     def new_tenant(self, tenant_name=None, tenant_description=None):
 395         """ Method adds a new tenant to VIM with this name.
 396             This action requires access to create VDC action in vCloud director.
 397
 398             Args:
 399                 tenant_name is tenant_name to be created.
 400                 tenant_description not used for this call
 401
 402             Return:
 403                 returns the tenant identifier in UUID format.
 404                 If action is failed method will throw vimconn.vimconnException method
 405             """
 406         vdc_task = self.create_vdc(vdc_name=tenant_name)
 407         if vdc_task is not None:
 408             vdc_uuid, value = vdc_task.popitem()
 409             self.logger.info("Created new vdc {} and uuid: {}".format(tenant_name, vdc_uuid))
 410             return vdc_uuid
 411         else:
 412             raise vimconn.vimconnException("Failed create tenant {}".format(tenant_name))
 413
 414     def delete_tenant(self, tenant_id=None):
 415         """ Delete a tenant from VIM
 416              Args:
 417                 tenant_id is tenant_id to be deleted.
 418
 419             Return:
 420                 returns the tenant identifier in UUID format.
 421                 If action is failed method will throw exception
 422         """
 423         vca = self.connect_as_admin()
 424         if not vca:
 425             raise vimconn.vimconnConnectionException("Failed to connect vCD")
 426
 427         if tenant_id is not None:
 428             if vca._session:
 429                 #Get OrgVDC
 430                 url_list = [self.url, '/api/vdc/', tenant_id]
 431                 orgvdc_herf = ''.join(url_list)
 432
 433                 headers = {'Accept':'application/*+xml;version=' + API_VERSION,
 434                            'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
 435                 response = self.perform_request(req_type='GET',
 436                                                 url=orgvdc_herf,
 437                                                 headers=headers)
 438
 439                 if response.status_code != requests.codes.ok:
 440                     self.logger.debug("delete_tenant():GET REST API call {} failed. "\
 441                                       "Return status code {}".format(orgvdc_herf,
 442                                                                      response.status_code))
 443                     raise vimconn.vimconnNotFoundException("Fail to get tenant {}".format(tenant_id))
 444
 445                 lxmlroot_respond = lxmlElementTree.fromstring(response.content)
 446                 namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.iteritems() if prefix}
 447                 #For python3
 448                 #namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.items() if prefix}
 449                 namespaces["xmlns"]= "http://www.vmware.com/vcloud/v1.5"
 450                 vdc_remove_href = lxmlroot_respond.find("xmlns:Link[@rel='remove']",namespaces).attrib['href']
 451                 vdc_remove_href = vdc_remove_href + '?recursive=true&force=true'
 452
 453                 response = self.perform_request(req_type='DELETE',
 454                                                 url=vdc_remove_href,
 455                                                 headers=headers)
 456
 457                 if response.status_code == 202:
 458                     time.sleep(5)
 459                     return tenant_id
 460                 else:
 461                     self.logger.debug("delete_tenant(): DELETE REST API call {} failed. "\
 462                                       "Return status code {}".format(vdc_remove_href,
 463                                                                      response.status_code))
 464                     raise vimconn.vimconnException("Fail to delete tenant with ID {}".format(tenant_id))
 465         else:
 466             self.logger.debug("delete_tenant():Incorrect tenant ID  {}".format(tenant_id))
 467             raise vimconn.vimconnNotFoundException("Fail to get tenant {}".format(tenant_id))
 468
 469
 470     def get_tenant_list(self, filter_dict={}):
 471         """Obtain tenants of VIM
 472         filter_dict can contain the following keys:
 473             name: filter by tenant name
 474             id: filter by tenant uuid/id
 475             <other VIM specific>
 476         Returns the tenant list of dictionaries:
 477             [{'name':'<name>, 'id':'<id>, ...}, ...]
 478
 479         """
 480         org_dict = self.get_org(self.org_uuid)
 481         vdcs_dict = org_dict['vdcs']
 482
 483         vdclist = []
 484         try:
 485             for k in vdcs_dict:
 486                 entry = {'name': vdcs_dict[k], 'id': k}
 487                 # if caller didn't specify dictionary we return all tenants.
 488                 if filter_dict is not None and filter_dict:
 489                     filtered_entry = entry.copy()
 490                     filtered_dict = set(entry.keys()) - set(filter_dict)
 491                     for unwanted_key in filtered_dict: del entry[unwanted_key]
 492                     if filter_dict == entry:
 493                         vdclist.append(filtered_entry)
 494                 else:
 495                     vdclist.append(entry)
 496         except:
 497             self.logger.debug("Error in get_tenant_list()")
 498             self.logger.debug(traceback.format_exc())
 499             raise vimconn.vimconnException("Incorrect state. {}")
 500
 501         return vdclist
 502
 503     def new_network(self, net_name, net_type, ip_profile=None, shared=False):
 504         """Adds a tenant network to VIM
 505             net_name is the name
 506             net_type can be 'bridge','data'.'ptp'.
 507             ip_profile is a dict containing the IP parameters of the network
 508             shared is a boolean
 509         Returns the network identifier"""
 510
 511         self.logger.debug("new_network tenant {} net_type {} ip_profile {} shared {}"
 512                           .format(net_name, net_type, ip_profile, shared))
 513
 514         isshared = 'false'
 515         if shared:
 516             isshared = 'true'
 517
 518 # ############# Stub code for SRIOV #################
 519 #         if net_type == "data" or net_type == "ptp":
 520 #             if self.config.get('dv_switch_name') == None:
 521 #                  raise vimconn.vimconnConflictException("You must provide 'dv_switch_name' at config value")
 522 #             network_uuid = self.create_dvPort_group(net_name)
 523
 524         network_uuid = self.create_network(network_name=net_name, net_type=net_type,
 525                                            ip_profile=ip_profile, isshared=isshared)
 526         if network_uuid is not None:
 527             return network_uuid
 528         else:
 529             raise vimconn.vimconnUnexpectedResponse("Failed create a new network {}".format(net_name))
 530
 531     def get_vcd_network_list(self):
 532         """ Method available organization for a logged in tenant
 533
 534             Returns:
 535                 The return vca object that letter can be used to connect to vcloud direct as admin
 536         """
 537
 538         self.logger.debug("get_vcd_network_list(): retrieving network list for vcd {}".format(self.tenant_name))
 539
 540         if not self.tenant_name:
 541             raise vimconn.vimconnConnectionException("Tenant name is empty.")
 542
 543         org, vdc = self.get_vdc_details()
 544         if vdc is None:
 545             raise vimconn.vimconnConnectionException("Can't retrieve information for a VDC {}".format(self.tenant_name))
 546
 547         vdc_uuid = vdc.get('id').split(":")[3]
 548         if self.client._session:
 549                 headers = {'Accept':'application/*+xml;version=' + API_VERSION,
 550                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
 551                 response = self.perform_request(req_type='GET',
 552                                            url=vdc.get('href'),
 553                                                headers=headers)
 554         if response.status_code != 200:
 555             self.logger.error("Failed to get vdc content")
 556             raise vimconn.vimconnNotFoundException("Failed to get vdc content")
 557         else:
 558             content = XmlElementTree.fromstring(response.content)
 559
 560         network_list = []
 561         try:
 562             for item in content:
 563                 if item.tag.split('}')[-1] == 'AvailableNetworks':
 564                     for net in item:
 565                         response = self.perform_request(req_type='GET',
 566                                                    url=net.get('href'),
 567                                                        headers=headers)
 568
 569                         if response.status_code != 200:
 570                             self.logger.error("Failed to get network content")
 571                             raise vimconn.vimconnNotFoundException("Failed to get network content")
 572                         else:
 573                             net_details = XmlElementTree.fromstring(response.content)
 574
 575                             filter_dict = {}
 576                             net_uuid = net_details.get('id').split(":")
 577                             if len(net_uuid) != 4:
 578                                 continue
 579                             else:
 580                                 net_uuid = net_uuid[3]
 581                                 # create dict entry
 582                                 self.logger.debug("Adding  {} to a list vcd id {} network {}".format(net_uuid,
 583                                                                                                         vdc_uuid,
 584                                                                                          net_details.get('name')))
 585                                 filter_dict["name"] = net_details.get('name')
 586                                 filter_dict["id"] = net_uuid
 587                                 if [i.text for i in net_details if i.tag.split('}')[-1] == 'IsShared'][0] == 'true':
 588                                     shared = True
 589                                 else:
 590                                     shared = False
 591                                 filter_dict["shared"] = shared
 592                                 filter_dict["tenant_id"] = vdc_uuid
 593                                 if int(net_details.get('status')) == 1:
 594                                     filter_dict["admin_state_up"] = True
 595                                 else:
 596                                     filter_dict["admin_state_up"] = False
 597                                 filter_dict["status"] = "ACTIVE"
 598                                 filter_dict["type"] = "bridge"
 599                                 network_list.append(filter_dict)
 600                                 self.logger.debug("get_vcd_network_list adding entry {}".format(filter_dict))
 601         except:
 602             self.logger.debug("Error in get_vcd_network_list", exc_info=True)
 603             pass
 604
 605         self.logger.debug("get_vcd_network_list returning {}".format(network_list))
 606         return network_list
 607
 608     def get_network_list(self, filter_dict={}):
 609         """Obtain tenant networks of VIM
 610         Filter_dict can be:
 611             name: network name  OR/AND
 612             id: network uuid    OR/AND
 613             shared: boolean     OR/AND
 614             tenant_id: tenant   OR/AND
 615             admin_state_up: boolean
 616             status: 'ACTIVE'
 617
 618         [{key : value , key : value}]
 619
 620         Returns the network list of dictionaries:
 621             [{<the fields at Filter_dict plus some VIM specific>}, ...]
 622             List can be empty
 623         """
 624
 625         self.logger.debug("get_network_list(): retrieving network list for vcd {}".format(self.tenant_name))
 626
 627         if not self.tenant_name:
 628             raise vimconn.vimconnConnectionException("Tenant name is empty.")
 629
 630         org, vdc = self.get_vdc_details()
 631         if vdc is None:
 632             raise vimconn.vimconnConnectionException("Can't retrieve information for a VDC {}.".format(self.tenant_name))
 633
 634         try:
 635             vdcid = vdc.get('id').split(":")[3]
 636
 637             if self.client._session:
 638                 headers = {'Accept':'application/*+xml;version=' + API_VERSION,
 639                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
 640                 response = self.perform_request(req_type='GET',
 641                                            url=vdc.get('href'),
 642                                                headers=headers)
 643             if response.status_code != 200:
 644                 self.logger.error("Failed to get vdc content")
 645                 raise vimconn.vimconnNotFoundException("Failed to get vdc content")
 646             else:
 647                 content = XmlElementTree.fromstring(response.content)
 648
 649             network_list = []
 650             for item in content:
 651                 if item.tag.split('}')[-1] == 'AvailableNetworks':
 652                     for net in item:
 653                         response = self.perform_request(req_type='GET',
 654                                                    url=net.get('href'),
 655                                                        headers=headers)
 656
 657                         if response.status_code != 200:
 658                             self.logger.error("Failed to get network content")
 659                             raise vimconn.vimconnNotFoundException("Failed to get network content")
 660                         else:
 661                             net_details = XmlElementTree.fromstring(response.content)
 662
 663                             filter_entry = {}
 664                             net_uuid = net_details.get('id').split(":")
 665                             if len(net_uuid) != 4:
 666                                 continue
 667                             else:
 668                                 net_uuid = net_uuid[3]
 669                                 # create dict entry
 670                                 self.logger.debug("Adding  {} to a list vcd id {} network {}".format(net_uuid,
 671                                                                                                         vdcid,
 672                                                                                          net_details.get('name')))
 673                                 filter_entry["name"] = net_details.get('name')
 674                                 filter_entry["id"] = net_uuid
 675                                 if [i.text for i in net_details if i.tag.split('}')[-1] == 'IsShared'][0] == 'true':
 676                                     shared = True
 677                                 else:
 678                                     shared = False
 679                                 filter_entry["shared"] = shared
 680                                 filter_entry["tenant_id"] = vdcid
 681                                 if int(net_details.get('status')) == 1:
 682                                     filter_entry["admin_state_up"] = True
 683                                 else:
 684                                     filter_entry["admin_state_up"] = False
 685                                 filter_entry["status"] = "ACTIVE"
 686                                 filter_entry["type"] = "bridge"
 687                                 filtered_entry = filter_entry.copy()
 688
 689                                 if filter_dict is not None and filter_dict:
 690                                     # we remove all the key : value we don't care and match only
 691                                     # respected field
 692                                     filtered_dict = set(filter_entry.keys()) - set(filter_dict)
 693                                     for unwanted_key in filtered_dict: del filter_entry[unwanted_key]
 694                                     if filter_dict == filter_entry:
 695                                         network_list.append(filtered_entry)
 696                                 else:
 697                                     network_list.append(filtered_entry)
 698         except Exception as e:
 699             self.logger.debug("Error in get_network_list",exc_info=True)
 700             if isinstance(e, vimconn.vimconnException):
 701                 raise
 702             else:
 703                 raise vimconn.vimconnNotFoundException("Failed : Networks list not found {} ".format(e))
 704
 705         self.logger.debug("Returning {}".format(network_list))
 706         return network_list
 707
 708     def get_network(self, net_id):
 709         """Method obtains network details of net_id VIM network
 710            Return a dict with  the fields at filter_dict (see get_network_list) plus some VIM specific>}, ...]"""
 711
 712         try:
 713             org, vdc = self.get_vdc_details()
 714             vdc_id = vdc.get('id').split(":")[3]
 715             if self.client._session:
 716                 headers = {'Accept':'application/*+xml;version=' + API_VERSION,
 717                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
 718                 response = self.perform_request(req_type='GET',
 719                                            url=vdc.get('href'),
 720                                                headers=headers)
 721             if response.status_code != 200:
 722                 self.logger.error("Failed to get vdc content")
 723                 raise vimconn.vimconnNotFoundException("Failed to get vdc content")
 724             else:
 725                 content = XmlElementTree.fromstring(response.content)
 726
 727             filter_dict = {}
 728
 729             for item in content:
 730                 if item.tag.split('}')[-1] == 'AvailableNetworks':
 731                     for net in item:
 732                         response = self.perform_request(req_type='GET',
 733                                                    url=net.get('href'),
 734                                                        headers=headers)
 735
 736                         if response.status_code != 200:
 737                             self.logger.error("Failed to get network content")
 738                             raise vimconn.vimconnNotFoundException("Failed to get network content")
 739                         else:
 740                             net_details = XmlElementTree.fromstring(response.content)
 741
 742                             vdc_network_id = net_details.get('id').split(":")
 743                             if len(vdc_network_id) == 4 and vdc_network_id[3] == net_id:
 744                                 filter_dict["name"] = net_details.get('name')
 745                                 filter_dict["id"] = vdc_network_id[3]
 746                                 if [i.text for i in net_details if i.tag.split('}')[-1] == 'IsShared'][0] == 'true':
 747                                     shared = True
 748                                 else:
 749                                     shared = False
 750                                 filter_dict["shared"] = shared
 751                                 filter_dict["tenant_id"] = vdc_id
 752                                 if int(net_details.get('status')) == 1:
 753                                     filter_dict["admin_state_up"] = True
 754                                 else:
 755                                     filter_dict["admin_state_up"] = False
 756                                 filter_dict["status"] = "ACTIVE"
 757                                 filter_dict["type"] = "bridge"
 758                                 self.logger.debug("Returning {}".format(filter_dict))
 759                                 return filter_dict
 760                     else:
 761                         raise vimconn.vimconnNotFoundException("Network {} not found".format(net_id))
 762         except Exception as e:
 763             self.logger.debug("Error in get_network")
 764             self.logger.debug(traceback.format_exc())
 765             if isinstance(e, vimconn.vimconnException):
 766                 raise
 767             else:
 768                 raise vimconn.vimconnNotFoundException("Failed : Network not found {} ".format(e))
 769
 770         return filter_dict
 771
 772     def delete_network(self, net_id):
 773         """
 774             Method Deletes a tenant network from VIM, provide the network id.
 775
 776             Returns the network identifier or raise an exception
 777         """
 778
 779         # ############# Stub code for SRIOV #################
 780 #         dvport_group = self.get_dvport_group(net_id)
 781 #         if dvport_group:
 782 #             #delete portgroup
 783 #             status = self.destroy_dvport_group(net_id)
 784 #             if status:
 785 #                 # Remove vlanID from persistent info
 786 #                 if net_id in self.persistent_info["used_vlanIDs"]:
 787 #                     del self.persistent_info["used_vlanIDs"][net_id]
 788 #
 789 #                 return net_id
 790
 791         vcd_network = self.get_vcd_network(network_uuid=net_id)
 792         if vcd_network is not None and vcd_network:
 793             if self.delete_network_action(network_uuid=net_id):
 794                 return net_id
 795         else:
 796             raise vimconn.vimconnNotFoundException("Network {} not found".format(net_id))
 797
 798     def refresh_nets_status(self, net_list):
 799         """Get the status of the networks
 800            Params: the list of network identifiers
 801            Returns a dictionary with:
 802                 net_id:         #VIM id of this network
 803                     status:     #Mandatory. Text with one of:
 804                                 #  DELETED (not found at vim)
 805                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 806                                 #  OTHER (Vim reported other status not understood)
 807                                 #  ERROR (VIM indicates an ERROR status)
 808                                 #  ACTIVE, INACTIVE, DOWN (admin down),
 809                                 #  BUILD (on building process)
 810                                 #
 811                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 812                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 813
 814         """
 815
 816         dict_entry = {}
 817         try:
 818             for net in net_list:
 819                 errormsg = ''
 820                 vcd_network = self.get_vcd_network(network_uuid=net)
 821                 if vcd_network is not None and vcd_network:
 822                     if vcd_network['status'] == '1':
 823                         status = 'ACTIVE'
 824                     else:
 825                         status = 'DOWN'
 826                 else:
 827                     status = 'DELETED'
 828                     errormsg = 'Network not found.'
 829
 830                 dict_entry[net] = {'status': status, 'error_msg': errormsg,
 831                                    'vim_info': yaml.safe_dump(vcd_network)}
 832         except:
 833             self.logger.debug("Error in refresh_nets_status")
 834             self.logger.debug(traceback.format_exc())
 835
 836         return dict_entry
 837
 838     def get_flavor(self, flavor_id):
 839         """Obtain flavor details from the  VIM
 840             Returns the flavor dict details {'id':<>, 'name':<>, other vim specific } #TODO to concrete
 841         """
 842         if flavor_id not in vimconnector.flavorlist:
 843             raise vimconn.vimconnNotFoundException("Flavor not found.")
 844         return vimconnector.flavorlist[flavor_id]
 845
 846     def new_flavor(self, flavor_data):
 847         """Adds a tenant flavor to VIM
 848             flavor_data contains a dictionary with information, keys:
 849                 name: flavor name
 850                 ram: memory (cloud type) in MBytes
 851                 vpcus: cpus (cloud type)
 852                 extended: EPA parameters
 853                   - numas: #items requested in same NUMA
 854                         memory: number of 1G huge pages memory
 855                         paired-threads|cores|threads: number of paired hyperthreads, complete cores OR individual threads
 856                         interfaces: # passthrough(PT) or SRIOV interfaces attached to this numa
 857                           - name: interface name
 858                             dedicated: yes|no|yes:sriov;  for PT, SRIOV or only one SRIOV for the physical NIC
 859                             bandwidth: X Gbps; requested guarantee bandwidth
 860                             vpci: requested virtual PCI address
 861                 disk: disk size
 862                 is_public:
 863                  #TODO to concrete
 864         Returns the flavor identifier"""
 865
 866         # generate a new uuid put to internal dict and return it.
 867         self.logger.debug("Creating new flavor - flavor_data: {}".format(flavor_data))
 868         new_flavor=flavor_data
 869         ram = flavor_data.get(FLAVOR_RAM_KEY, 1024)
 870         cpu = flavor_data.get(FLAVOR_VCPUS_KEY, 1)
 871         disk = flavor_data.get(FLAVOR_DISK_KEY, 0)
 872
 873         if not isinstance(ram, int):
 874             raise vimconn.vimconnException("Non-integer value for ram")
 875         elif not isinstance(cpu, int):
 876             raise vimconn.vimconnException("Non-integer value for cpu")
 877         elif not isinstance(disk, int):
 878             raise vimconn.vimconnException("Non-integer value for disk")
 879
 880         extended_flv = flavor_data.get("extended")
 881         if extended_flv:
 882             numas=extended_flv.get("numas")
 883             if numas:
 884                 for numa in numas:
 885                     #overwrite ram and vcpus
 886                     if 'memory' in numa:
 887                         ram = numa['memory']*1024
 888                     if 'paired-threads' in numa:
 889                         cpu = numa['paired-threads']*2
 890                     elif 'cores' in numa:
 891                         cpu = numa['cores']
 892                     elif 'threads' in numa:
 893                         cpu = numa['threads']
 894
 895         new_flavor[FLAVOR_RAM_KEY] = ram
 896         new_flavor[FLAVOR_VCPUS_KEY] = cpu
 897         new_flavor[FLAVOR_DISK_KEY] = disk
 898         # generate a new uuid put to internal dict and return it.
 899         flavor_id = uuid.uuid4()
 900         vimconnector.flavorlist[str(flavor_id)] = new_flavor
 901         self.logger.debug("Created flavor - {} : {}".format(flavor_id, new_flavor))
 902
 903         return str(flavor_id)
 904
 905     def delete_flavor(self, flavor_id):
 906         """Deletes a tenant flavor from VIM identify by its id
 907
 908            Returns the used id or raise an exception
 909         """
 910         if flavor_id not in vimconnector.flavorlist:
 911             raise vimconn.vimconnNotFoundException("Flavor not found.")
 912
 913         vimconnector.flavorlist.pop(flavor_id, None)
 914         return flavor_id
 915
 916     def new_image(self, image_dict):
 917         """
 918         Adds a tenant image to VIM
 919         Returns:
 920             200, image-id        if the image is created
 921             <0, message          if there is an error
 922         """
 923
 924         return self.get_image_id_from_path(image_dict['location'])
 925
 926     def delete_image(self, image_id):
 927         """
 928             Deletes a tenant image from VIM
 929             Args:
 930                 image_id is ID of Image to be deleted
 931             Return:
 932                 returns the image identifier in UUID format or raises an exception on error
 933         """
 934         conn = self.connect_as_admin()
 935         if not conn:
 936             raise vimconn.vimconnConnectionException("Failed to connect vCD")
 937         # Get Catalog details
 938         url_list = [self.url, '/api/catalog/', image_id]
 939         catalog_herf = ''.join(url_list)
 940
 941         headers = {'Accept':'application/*+xml;version=' + API_VERSION,
 942                   'x-vcloud-authorization': conn._session.headers['x-vcloud-authorization']}
 943
 944         response = self.perform_request(req_type='GET',
 945                                         url=catalog_herf,
 946                                         headers=headers)
 947
 948         if response.status_code != requests.codes.ok:
 949             self.logger.debug("delete_image():GET REST API call {} failed. "\
 950                               "Return status code {}".format(catalog_herf,
 951                                                              response.status_code))
 952             raise vimconn.vimconnNotFoundException("Fail to get image {}".format(image_id))
 953
 954         lxmlroot_respond = lxmlElementTree.fromstring(response.content)
 955         namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.iteritems() if prefix}
 956         #For python3
 957         #namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.items() if prefix}
 958         namespaces["xmlns"]= "http://www.vmware.com/vcloud/v1.5"
 959
 960         catalogItems_section = lxmlroot_respond.find("xmlns:CatalogItems",namespaces)
 961         catalogItems = catalogItems_section.iterfind("xmlns:CatalogItem",namespaces)
 962         for catalogItem in catalogItems:
 963             catalogItem_href = catalogItem.attrib['href']
 964
 965             response = self.perform_request(req_type='GET',
 966                                         url=catalogItem_href,
 967                                         headers=headers)
 968
 969             if response.status_code != requests.codes.ok:
 970                 self.logger.debug("delete_image():GET REST API call {} failed. "\
 971                                   "Return status code {}".format(catalog_herf,
 972                                                                  response.status_code))
 973                 raise vimconn.vimconnNotFoundException("Fail to get catalogItem {} for catalog {}".format(
 974                                                                                     catalogItem,
 975                                                                                     image_id))
 976
 977             lxmlroot_respond = lxmlElementTree.fromstring(response.content)
 978             namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.iteritems() if prefix}
 979             #For python3
 980             #namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.items() if prefix}
 981             namespaces["xmlns"]= "http://www.vmware.com/vcloud/v1.5"
 982             catalogitem_remove_href = lxmlroot_respond.find("xmlns:Link[@rel='remove']",namespaces).attrib['href']
 983
 984             #Remove catalogItem
 985             response = self.perform_request(req_type='DELETE',
 986                                         url=catalogitem_remove_href,
 987                                         headers=headers)
 988             if response.status_code == requests.codes.no_content:
 989                 self.logger.debug("Deleted Catalog item {}".format(catalogItem))
 990             else:
 991                 raise vimconn.vimconnException("Fail to delete Catalog Item {}".format(catalogItem))
 992
 993         #Remove catalog
 994         url_list = [self.url, '/api/admin/catalog/', image_id]
 995         catalog_remove_herf = ''.join(url_list)
 996         response = self.perform_request(req_type='DELETE',
 997                                         url=catalog_remove_herf,
 998                                         headers=headers)
 999
1000         if response.status_code == requests.codes.no_content:
1001             self.logger.debug("Deleted Catalog {}".format(image_id))
1002             return image_id
1003         else:
1004             raise vimconn.vimconnException("Fail to delete Catalog {}".format(image_id))
1005
1006
1007     def catalog_exists(self, catalog_name, catalogs):
1008         """
1009
1010         :param catalog_name:
1011         :param catalogs:
1012         :return:
1013         """
1014         for catalog in catalogs:
1015             if catalog['name'] == catalog_name:
1016                 return True
1017         return False
1018
1019     def create_vimcatalog(self, vca=None, catalog_name=None):
1020         """ Create new catalog entry in vCloud director.
1021
1022             Args
1023                 vca:  vCloud director.
1024                 catalog_name catalog that client wish to create.   Note no validation done for a name.
1025                 Client must make sure that provide valid string representation.
1026
1027              Return (bool) True if catalog created.
1028
1029         """
1030         try:
1031             result = vca.create_catalog(catalog_name, catalog_name)
1032             if result is not None:
1033                 return True
1034             catalogs = vca.list_catalogs()
1035         except:
1036             return False
1037         return self.catalog_exists(catalog_name, catalogs)
1038
1039     # noinspection PyIncorrectDocstring
1040     def upload_ovf(self, vca=None, catalog_name=None, image_name=None, media_file_name=None,
1041                    description='', progress=False, chunk_bytes=128 * 1024):
1042         """
1043         Uploads a OVF file to a vCloud catalog
1044
1045         :param chunk_bytes:
1046         :param progress:
1047         :param description:
1048         :param image_name:
1049         :param vca:
1050         :param catalog_name: (str): The name of the catalog to upload the media.
1051         :param media_file_name: (str): The name of the local media file to upload.
1052         :return: (bool) True if the media file was successfully uploaded, false otherwise.
1053         """
1054         os.path.isfile(media_file_name)
1055         statinfo = os.stat(media_file_name)
1056
1057         #  find a catalog entry where we upload OVF.
1058         #  create vApp Template and check the status if vCD able to read OVF it will respond with appropirate
1059         #  status change.
1060         #  if VCD can parse OVF we upload VMDK file
1061         try:
1062             for catalog in vca.list_catalogs():
1063                 if catalog_name != catalog['name']:
1064                     continue
1065                 catalog_href = "{}/api/catalog/{}/action/upload".format(self.url, catalog['id'])
1066                 data = """
1067                 <UploadVAppTemplateParams name="{}" xmlns="http://www.vmware.com/vcloud/v1.5" xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/1"><Description>{} vApp Template</Description></UploadVAppTemplateParams>
1068                 """.format(catalog_name, description)
1069
1070                 if self.client:
1071                     headers = {'Accept':'application/*+xml;version=' + API_VERSION,
1072                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
1073                     headers['Content-Type'] = 'application/vnd.vmware.vcloud.uploadVAppTemplateParams+xml'
1074
1075                 response = self.perform_request(req_type='POST',
1076                                                 url=catalog_href,
1077                                                 headers=headers,
1078                                                 data=data)
1079
1080                 if response.status_code == requests.codes.created:
1081                     catalogItem = XmlElementTree.fromstring(response.content)
1082                     entity = [child for child in catalogItem if
1083                               child.get("type") == "application/vnd.vmware.vcloud.vAppTemplate+xml"][0]
1084                     href = entity.get('href')
1085                     template = href
1086
1087                     response = self.perform_request(req_type='GET',
1088                                                     url=href,
1089                                                     headers=headers)
1090
1091                     if response.status_code == requests.codes.ok:
1092                         headers['Content-Type'] = 'Content-Type text/xml'
1093                         result = re.search('rel="upload:default"\shref="(.*?\/descriptor.ovf)"',response.content)
1094                         if result:
1095                             transfer_href = result.group(1)
1096
1097                         response = self.perform_request(req_type='PUT',
1098                                                     url=transfer_href,
1099                                                     headers=headers,
1100                                                     data=open(media_file_name, 'rb'))
1101                         if response.status_code != requests.codes.ok:
1102                             self.logger.debug(
1103                                 "Failed create vApp template for catalog name {} and image {}".format(catalog_name,
1104                                                                                                       media_file_name))
1105                             return False
1106
1107                     # TODO fix this with aync block
1108                     time.sleep(5)
1109
1110                     self.logger.debug("vApp template for catalog name {} and image {}".format(catalog_name, media_file_name))
1111
1112                     # uploading VMDK file
1113                     # check status of OVF upload and upload remaining files.
1114                     response = self.perform_request(req_type='GET',
1115                                                     url=template,
1116                                                     headers=headers)
1117
1118                     if response.status_code == requests.codes.ok:
1119                         result = re.search('rel="upload:default"\s*href="(.*?vmdk)"',response.content)
1120                         if result:
1121                             link_href = result.group(1)
1122                         # we skip ovf since it already uploaded.
1123                         if 'ovf' in link_href:
1124                             continue
1125                         # The OVF file and VMDK must be in a same directory
1126                         head, tail = os.path.split(media_file_name)
1127                         file_vmdk = head + '/' + link_href.split("/")[-1]
1128                         if not os.path.isfile(file_vmdk):
1129                             return False
1130                         statinfo = os.stat(file_vmdk)
1131                         if statinfo.st_size == 0:
1132                             return False
1133                         hrefvmdk = link_href
1134
1135                         if progress:
1136                             widgets = ['Uploading file: ', Percentage(), ' ', Bar(), ' ', ETA(), ' ',
1137                                            FileTransferSpeed()]
1138                             progress_bar = ProgressBar(widgets=widgets, maxval=statinfo.st_size).start()
1139
1140                         bytes_transferred = 0
1141                         f = open(file_vmdk, 'rb')
1142                         while bytes_transferred < statinfo.st_size:
1143                             my_bytes = f.read(chunk_bytes)
1144                             if len(my_bytes) <= chunk_bytes:
1145                                 headers['Content-Range'] = 'bytes %s-%s/%s' % (
1146                                     bytes_transferred, len(my_bytes) - 1, statinfo.st_size)
1147                                 headers['Content-Length'] = str(len(my_bytes))
1148                                 response = requests.put(url=hrefvmdk,
1149                                                          headers=headers,
1150                                                          data=my_bytes,
1151                                                          verify=False)
1152                                 if response.status_code == requests.codes.ok:
1153                                     bytes_transferred += len(my_bytes)
1154                                     if progress:
1155                                         progress_bar.update(bytes_transferred)
1156                                 else:
1157                                     self.logger.debug(
1158                                         'file upload failed with error: [%s] %s' % (response.status_code,
1159                                                                                         response.content))
1160
1161                                     f.close()
1162                                     return False
1163                         f.close()
1164                         if progress:
1165                             progress_bar.finish()
1166                             time.sleep(10)
1167                     return True
1168                 else:
1169                     self.logger.debug("Failed retrieve vApp template for catalog name {} for OVF {}".
1170                                       format(catalog_name, media_file_name))
1171                     return False
1172         except Exception as exp:
1173             self.logger.debug("Failed while uploading OVF to catalog {} for OVF file {} with Exception {}"
1174                 .format(catalog_name,media_file_name, exp))
1175             raise vimconn.vimconnException(
1176                 "Failed while uploading OVF to catalog {} for OVF file {} with Exception {}"
1177                 .format(catalog_name,media_file_name, exp))
1178
1179         self.logger.debug("Failed retrieve catalog name {} for OVF file {}".format(catalog_name, media_file_name))
1180         return False
1181
1182     def upload_vimimage(self, vca=None, catalog_name=None, media_name=None, medial_file_name=None, progress=False):
1183         """Upload media file"""
1184         # TODO add named parameters for readability
1185
1186         return self.upload_ovf(vca=vca, catalog_name=catalog_name, image_name=media_name.split(".")[0],
1187                                media_file_name=medial_file_name, description='medial_file_name', progress=progress)
1188
1189     def validate_uuid4(self, uuid_string=None):
1190         """  Method validate correct format of UUID.
1191
1192         Return: true if string represent valid uuid
1193         """
1194         try:
1195             val = uuid.UUID(uuid_string, version=4)
1196         except ValueError:
1197             return False
1198         return True
1199
1200     def get_catalogid(self, catalog_name=None, catalogs=None):
1201         """  Method check catalog and return catalog ID in UUID format.
1202
1203         Args
1204             catalog_name: catalog name as string
1205             catalogs:  list of catalogs.
1206
1207         Return: catalogs uuid
1208         """
1209
1210         for catalog in catalogs:
1211             if catalog['name'] == catalog_name:
1212                 catalog_id = catalog['id']
1213                 return catalog_id
1214         return None
1215
1216     def get_catalogbyid(self, catalog_uuid=None, catalogs=None):
1217         """  Method check catalog and return catalog name lookup done by catalog UUID.
1218
1219         Args
1220             catalog_name: catalog name as string
1221             catalogs:  list of catalogs.
1222
1223         Return: catalogs name or None
1224         """
1225
1226         if not self.validate_uuid4(uuid_string=catalog_uuid):
1227             return None
1228
1229         for catalog in catalogs:
1230             catalog_id = catalog.get('id')
1231             if catalog_id == catalog_uuid:
1232                 return catalog.get('name')
1233         return None
1234
1235     def get_catalog_obj(self, catalog_uuid=None, catalogs=None):
1236         """  Method check catalog and return catalog name lookup done by catalog UUID.
1237
1238         Args
1239             catalog_name: catalog name as string
1240             catalogs:  list of catalogs.
1241
1242         Return: catalogs name or None
1243         """
1244
1245         if not self.validate_uuid4(uuid_string=catalog_uuid):
1246             return None
1247
1248         for catalog in catalogs:
1249             catalog_id = catalog.get('id')
1250             if catalog_id == catalog_uuid:
1251                 return catalog
1252         return None
1253
1254     def get_image_id_from_path(self, path=None, progress=False):
1255         """  Method upload OVF image to vCloud director.
1256
1257         Each OVF image represented as single catalog entry in vcloud director.
1258         The method check for existing catalog entry.  The check done by file name without file extension.
1259
1260         if given catalog name already present method will respond with existing catalog uuid otherwise
1261         it will create new catalog entry and upload OVF file to newly created catalog.
1262
1263         If method can't create catalog entry or upload a file it will throw exception.
1264
1265         Method accept boolean flag progress that will output progress bar. It useful method
1266         for standalone upload use case. In case to test large file upload.
1267
1268         Args
1269             path: - valid path to OVF file.
1270             progress - boolean progress bar show progress bar.
1271
1272         Return: if image uploaded correct method will provide image catalog UUID.
1273         """
1274
1275         if not path:
1276             raise vimconn.vimconnException("Image path can't be None.")
1277
1278         if not os.path.isfile(path):
1279             raise vimconn.vimconnException("Can't read file. File not found.")
1280
1281         if not os.access(path, os.R_OK):
1282             raise vimconn.vimconnException("Can't read file. Check file permission to read.")
1283
1284         self.logger.debug("get_image_id_from_path() client requesting {} ".format(path))
1285
1286         dirpath, filename = os.path.split(path)
1287         flname, file_extension = os.path.splitext(path)
1288         if file_extension != '.ovf':
1289             self.logger.debug("Wrong file extension {} connector support only OVF container.".format(file_extension))
1290             raise vimconn.vimconnException("Wrong container.  vCloud director supports only OVF.")
1291
1292         catalog_name = os.path.splitext(filename)[0]
1293         catalog_md5_name = hashlib.md5(path).hexdigest()
1294         self.logger.debug("File name {} Catalog Name {} file path {} "
1295                           "vdc catalog name {}".format(filename, catalog_name, path, catalog_md5_name))
1296
1297         try:
1298             org,vdc = self.get_vdc_details()
1299             catalogs = org.list_catalogs()
1300         except Exception as exp:
1301             self.logger.debug("Failed get catalogs() with Exception {} ".format(exp))
1302             raise vimconn.vimconnException("Failed get catalogs() with Exception {} ".format(exp))
1303
1304         if len(catalogs) == 0:
1305             self.logger.info("Creating a new catalog entry {} in vcloud director".format(catalog_name))
1306             result = self.create_vimcatalog(org, catalog_md5_name)
1307             if not result:
1308                 raise vimconn.vimconnException("Failed create new catalog {} ".format(catalog_md5_name))
1309
1310             result = self.upload_vimimage(vca=org, catalog_name=catalog_md5_name,
1311                                           media_name=filename, medial_file_name=path, progress=progress)
1312             if not result:
1313                 raise vimconn.vimconnException("Failed create vApp template for catalog {} ".format(catalog_name))
1314             return self.get_catalogid(catalog_name, catalogs)
1315         else:
1316             for catalog in catalogs:
1317                 # search for existing catalog if we find same name we return ID
1318                 # TODO optimize this
1319                 if catalog['name'] == catalog_md5_name:
1320                     self.logger.debug("Found existing catalog entry for {} "
1321                                       "catalog id {}".format(catalog_name,
1322                                                              self.get_catalogid(catalog_md5_name, catalogs)))
1323                     return self.get_catalogid(catalog_md5_name, catalogs)
1324
1325         # if we didn't find existing catalog we create a new one and upload image.
1326         self.logger.debug("Creating new catalog entry {} - {}".format(catalog_name, catalog_md5_name))
1327         result = self.create_vimcatalog(org, catalog_md5_name)
1328         if not result:
1329             raise vimconn.vimconnException("Failed create new catalog {} ".format(catalog_md5_name))
1330
1331         result = self.upload_vimimage(vca=org, catalog_name=catalog_md5_name,
1332                                       media_name=filename, medial_file_name=path, progress=progress)
1333         if not result:
1334             raise vimconn.vimconnException("Failed create vApp template for catalog {} ".format(catalog_md5_name))
1335
1336         return self.get_catalogid(catalog_md5_name, org.list_catalogs())
1337
1338     def get_image_list(self, filter_dict={}):
1339         '''Obtain tenant images from VIM
1340         Filter_dict can be:
1341             name: image name
1342             id: image uuid
1343             checksum: image checksum
1344             location: image path
1345         Returns the image list of dictionaries:
1346             [{<the fields at Filter_dict plus some VIM specific>}, ...]
1347             List can be empty
1348         '''
1349
1350         try:
1351             org, vdc = self.get_vdc_details()
1352             image_list = []
1353             catalogs = org.list_catalogs()
1354             if len(catalogs) == 0:
1355                 return image_list
1356             else:
1357                 for catalog in catalogs:
1358                     catalog_uuid = catalog.get('id')
1359                     name = catalog.get('name')
1360                     filtered_dict = {}
1361                     if filter_dict.get("name") and filter_dict["name"] != name:
1362                         continue
1363                     if filter_dict.get("id") and filter_dict["id"] != catalog_uuid:
1364                         continue
1365                     filtered_dict ["name"] = name
1366                     filtered_dict ["id"] = catalog_uuid
1367                     image_list.append(filtered_dict)
1368
1369                 self.logger.debug("List of already created catalog items: {}".format(image_list))
1370                 return image_list
1371         except Exception as exp:
1372             raise vimconn.vimconnException("Exception occured while retriving catalog items {}".format(exp))
1373
1374     def get_vappid(self, vdc=None, vapp_name=None):
1375         """ Method takes vdc object and vApp name and returns vapp uuid or None
1376
1377         Args:
1378             vdc: The VDC object.
1379             vapp_name: is application vappp name identifier
1380
1381         Returns:
1382                 The return vApp name otherwise None
1383         """
1384         if vdc is None or vapp_name is None:
1385             return None
1386         # UUID has following format https://host/api/vApp/vapp-30da58a3-e7c7-4d09-8f68-d4c8201169cf
1387         try:
1388             refs = filter(lambda ref: ref.name == vapp_name and ref.type_ == 'application/vnd.vmware.vcloud.vApp+xml',
1389                           vdc.ResourceEntities.ResourceEntity)
1390             #For python3
1391             #refs = [ref for ref in vdc.ResourceEntities.ResourceEntity\
1392             #         if ref.name == vapp_name and ref.type_ == 'application/vnd.vmware.vcloud.vApp+xml']
1393             if len(refs) == 1:
1394                 return refs[0].href.split("vapp")[1][1:]
1395         except Exception as e:
1396             self.logger.exception(e)
1397             return False
1398         return None
1399
1400     def check_vapp(self, vdc=None, vapp_uuid=None):
1401         """ Method Method returns True or False if vapp deployed in vCloud director
1402
1403             Args:
1404                 vca: Connector to VCA
1405                 vdc: The VDC object.
1406                 vappid: vappid is application identifier
1407
1408             Returns:
1409                 The return True if vApp deployed
1410                 :param vdc:
1411                 :param vapp_uuid:
1412         """
1413         try:
1414             refs = filter(lambda ref:
1415                           ref.type_ == 'application/vnd.vmware.vcloud.vApp+xml',
1416                           vdc.ResourceEntities.ResourceEntity)
1417             #For python3
1418             #refs = [ref for ref in vdc.ResourceEntities.ResourceEntity\
1419             #         if ref.type_ == 'application/vnd.vmware.vcloud.vApp+xml']
1420             for ref in refs:
1421                 vappid = ref.href.split("vapp")[1][1:]
1422                 # find vapp with respected vapp uuid
1423                 if vappid == vapp_uuid:
1424                     return True
1425         except Exception as e:
1426             self.logger.exception(e)
1427             return False
1428         return False
1429
1430     def get_namebyvappid(self, vapp_uuid=None):
1431         """Method returns vApp name from vCD and lookup done by vapp_id.
1432
1433         Args:
1434             vapp_uuid: vappid is application identifier
1435
1436         Returns:
1437             The return vApp name otherwise None
1438         """
1439         try:
1440             if self.client and vapp_uuid:
1441                 vapp_call = "{}/api/vApp/vapp-{}".format(self.url, vapp_uuid)
1442                 headers = {'Accept':'application/*+xml;version=' + API_VERSION,
1443                      'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
1444
1445                 response = self.perform_request(req_type='GET',
1446                                                 url=vapp_call,
1447                                                 headers=headers)
1448                 #Retry login if session expired & retry sending request
1449                 if response.status_code == 403:
1450                     response = self.retry_rest('GET', vapp_call)
1451
1452                 tree = XmlElementTree.fromstring(response.content)
1453                 return tree.attrib['name']
1454         except Exception as e:
1455             self.logger.exception(e)
1456             return None
1457         return None
1458
1459     def new_vminstance(self, name=None, description="", start=False, image_id=None, flavor_id=None, net_list=[],
1460                        cloud_config=None, disk_list=None, availability_zone_index=None, availability_zone_list=None):
1461         """Adds a VM instance to VIM
1462         Params:
1463             'start': (boolean) indicates if VM must start or created in pause mode.
1464             'image_id','flavor_id': image and flavor VIM id to use for the VM
1465             'net_list': list of interfaces, each one is a dictionary with:
1466                 'name': (optional) name for the interface.
1467                 'net_id': VIM network id where this interface must be connect to. Mandatory for type==virtual
1468                 'vpci': (optional) virtual vPCI address to assign at the VM. Can be ignored depending on VIM capabilities
1469                 'model': (optional and only have sense for type==virtual) interface model: virtio, e2000, ...
1470                 'mac_address': (optional) mac address to assign to this interface
1471                 #TODO: CHECK if an optional 'vlan' parameter is needed for VIMs when type if VF and net_id is not provided,
1472                     the VLAN tag to be used. In case net_id is provided, the internal network vlan is used for tagging VF
1473                 'type': (mandatory) can be one of:
1474                     'virtual', in this case always connected to a network of type 'net_type=bridge'
1475                      'PCI-PASSTHROUGH' or 'PF' (passthrough): depending on VIM capabilities it can be connected to a data/ptp network ot it
1476                            can created unconnected
1477                      'SR-IOV' or 'VF' (SRIOV with VLAN tag): same as PF for network connectivity.
1478                      'VFnotShared'(SRIOV without VLAN tag) same as PF for network connectivity. VF where no other VFs
1479                             are allocated on the same physical NIC
1480                 'bw': (optional) only for PF/VF/VFnotShared. Minimal Bandwidth required for the interface in GBPS
1481                 'port_security': (optional) If False it must avoid any traffic filtering at this interface. If missing
1482                                 or True, it must apply the default VIM behaviour
1483                 After execution the method will add the key:
1484                 'vim_id': must be filled/added by this method with the VIM identifier generated by the VIM for this
1485                         interface. 'net_list' is modified
1486             'cloud_config': (optional) dictionary with:
1487                 'key-pairs': (optional) list of strings with the public key to be inserted to the default user
1488                 'users': (optional) list of users to be inserted, each item is a dict with:
1489                     'name': (mandatory) user name,
1490                     'key-pairs': (optional) list of strings with the public key to be inserted to the user
1491                 'user-data': (optional) can be a string with the text script to be passed directly to cloud-init,
1492                     or a list of strings, each one contains a script to be passed, usually with a MIMEmultipart file
1493                 'config-files': (optional). List of files to be transferred. Each item is a dict with:
1494                     'dest': (mandatory) string with the destination absolute path
1495                     'encoding': (optional, by default text). Can be one of:
1496                         'b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64'
1497                     'content' (mandatory): string with the content of the file
1498                     'permissions': (optional) string with file permissions, typically octal notation '0644'
1499                     'owner': (optional) file owner, string with the format 'owner:group'
1500                 'boot-data-drive': boolean to indicate if user-data must be passed using a boot drive (hard disk)
1501             'disk_list': (optional) list with additional disks to the VM. Each item is a dict with:
1502                 'image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted
1503                 'size': (mandatory) string with the size of the disk in GB
1504             availability_zone_index: Index of availability_zone_list to use for this this VM. None if not AV required
1505             availability_zone_list: list of availability zones given by user in the VNFD descriptor.  Ignore if
1506                 availability_zone_index is None
1507         Returns a tuple with the instance identifier and created_items or raises an exception on error
1508             created_items can be None or a dictionary where this method can include key-values that will be passed to
1509             the method delete_vminstance and action_vminstance. Can be used to store created ports, volumes, etc.
1510             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
1511             as not present.
1512         """
1513         self.logger.info("Creating new instance for entry {}".format(name))
1514         self.logger.debug("desc {} boot {} image_id: {} flavor_id: {} net_list: {} cloud_config {} disk_list {} "\
1515                           "availability_zone_index {} availability_zone_list {}"\
1516                           .format(description, start, image_id, flavor_id, net_list, cloud_config, disk_list,\
1517                                   availability_zone_index, availability_zone_list))
1518
1519         #new vm name = vmname + tenant_id + uuid
1520         new_vm_name = [name, '-', str(uuid.uuid4())]
1521         vmname_andid = ''.join(new_vm_name)
1522
1523         for net in net_list:
1524             if net['type'] == "SR-IOV" or net['type'] == "PCI-PASSTHROUGH":
1525                 raise vimconn.vimconnNotSupportedException(
1526                       "Current vCD version does not support type : {}".format(net['type']))
1527
1528         if len(net_list) > 10:
1529             raise vimconn.vimconnNotSupportedException(
1530                       "The VM hardware versions 7 and above support upto 10 NICs only")
1531
1532         # if vm already deployed we return existing uuid
1533         # we check for presence of VDC, Catalog entry and Flavor.
1534         org, vdc = self.get_vdc_details()
1535         if vdc is None:
1536             raise vimconn.vimconnNotFoundException(
1537                 "new_vminstance(): Failed create vApp {}: (Failed retrieve VDC information)".format(name))
1538         catalogs = org.list_catalogs()
1539         if catalogs is None:
1540             #Retry once, if failed by refreshing token
1541             self.get_token()
1542             org = Org(self.client, resource=self.client.get_org())
1543             catalogs = org.list_catalogs()
1544         if catalogs is None:
1545             raise vimconn.vimconnNotFoundException(
1546                 "new_vminstance(): Failed create vApp {}: (Failed retrieve catalogs list)".format(name))
1547
1548         catalog_hash_name = self.get_catalogbyid(catalog_uuid=image_id, catalogs=catalogs)
1549         if catalog_hash_name:
1550             self.logger.info("Found catalog entry {} for image id {}".format(catalog_hash_name, image_id))
1551         else:
1552             raise vimconn.vimconnNotFoundException("new_vminstance(): Failed create vApp {}: "
1553                                                    "(Failed retrieve catalog information {})".format(name, image_id))
1554
1555
1556         # Set vCPU and Memory based on flavor.
1557         vm_cpus = None
1558         vm_memory = None
1559         vm_disk = None
1560         numas = None
1561
1562         if flavor_id is not None:
1563             if flavor_id not in vimconnector.flavorlist:
1564                 raise vimconn.vimconnNotFoundException("new_vminstance(): Failed create vApp {}: "
1565                                                        "Failed retrieve flavor information "
1566                                                        "flavor id {}".format(name, flavor_id))
1567             else:
1568                 try:
1569                     flavor = vimconnector.flavorlist[flavor_id]
1570                     vm_cpus = flavor[FLAVOR_VCPUS_KEY]
1571                     vm_memory = flavor[FLAVOR_RAM_KEY]
1572                     vm_disk = flavor[FLAVOR_DISK_KEY]
1573                     extended = flavor.get("extended", None)
1574                     if extended:
1575                         numas=extended.get("numas", None)
1576
1577                 except Exception as exp:
1578                     raise vimconn.vimconnException("Corrupted flavor. {}.Exception: {}".format(flavor_id, exp))
1579
1580         # image upload creates template name as catalog name space Template.
1581         templateName = self.get_catalogbyid(catalog_uuid=image_id, catalogs=catalogs)
1582         power_on = 'false'
1583         if start:
1584             power_on = 'true'
1585
1586         # client must provide at least one entry in net_list if not we report error
1587         #If net type is mgmt, then configure it as primary net & use its NIC index as primary NIC
1588         #If no mgmt, then the 1st NN in netlist is considered as primary net.
1589         primary_net = None
1590         primary_netname = None
1591         network_mode = 'bridged'
1592         if net_list is not None and len(net_list) > 0:
1593             for net in net_list:
1594                 if 'use' in net and net['use'] == 'mgmt' and not primary_net:
1595                     primary_net = net
1596             if primary_net is None:
1597                 primary_net = net_list[0]
1598
1599             try:
1600                 primary_net_id = primary_net['net_id']
1601                 network_dict = self.get_vcd_network(network_uuid=primary_net_id)
1602                 if 'name' in network_dict:
1603                     primary_netname = network_dict['name']
1604
1605             except KeyError:
1606                 raise vimconn.vimconnException("Corrupted flavor. {}".format(primary_net))
1607         else:
1608             raise vimconn.vimconnUnexpectedResponse("new_vminstance(): Failed network list is empty.".format(name))
1609
1610         # use: 'data', 'bridge', 'mgmt'
1611         # create vApp.  Set vcpu and ram based on flavor id.
1612         try:
1613             vdc_obj = VDC(self.client, resource=org.get_vdc(self.tenant_name))
1614             if not vdc_obj:
1615                 raise vimconn.vimconnNotFoundException("new_vminstance(): Failed to get VDC object")
1616
1617             for retry in (1,2):
1618                 items = org.get_catalog_item(catalog_hash_name, catalog_hash_name)
1619                 catalog_items = [items.attrib]
1620
1621                 if len(catalog_items) == 1:
1622                     if self.client:
1623                         headers = {'Accept':'application/*+xml;version=' + API_VERSION,
1624                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
1625
1626                     response = self.perform_request(req_type='GET',
1627                                                 url=catalog_items[0].get('href'),
1628                                                 headers=headers)
1629                     catalogItem = XmlElementTree.fromstring(response.content)
1630                     entity = [child for child in catalogItem if child.get("type") == "application/vnd.vmware.vcloud.vAppTemplate+xml"][0]
1631                     vapp_tempalte_href = entity.get("href")
1632
1633                 response = self.perform_request(req_type='GET',
1634                                                     url=vapp_tempalte_href,
1635                                                     headers=headers)
1636                 if response.status_code != requests.codes.ok:
1637                     self.logger.debug("REST API call {} failed. Return status code {}".format(vapp_tempalte_href,
1638                                                                                            response.status_code))
1639                 else:
1640                     result = (response.content).replace("\n"," ")
1641
1642                 src = re.search('<Vm goldMaster="false"\sstatus="\d+"\sname="(.*?)"\s'
1643                                                'id="(\w+:\w+:vm:.*?)"\shref="(.*?)"\s'
1644                               'type="application/vnd\.vmware\.vcloud\.vm\+xml',result)
1645                 if src:
1646                     vm_name = src.group(1)
1647                     vm_id = src.group(2)
1648                     vm_href = src.group(3)
1649
1650                 cpus = re.search('<rasd:Description>Number of Virtual CPUs</.*?>(\d+)</rasd:VirtualQuantity>',result).group(1)
1651                 memory_mb = re.search('<rasd:Description>Memory Size</.*?>(\d+)</rasd:VirtualQuantity>',result).group(1)
1652                 cores = re.search('<vmw:CoresPerSocket ovf:required.*?>(\d+)</vmw:CoresPerSocket>',result).group(1)
1653
1654                 headers['Content-Type'] = 'application/vnd.vmware.vcloud.instantiateVAppTemplateParams+xml'
1655                 vdc_id = vdc.get('id').split(':')[-1]
1656                 instantiate_vapp_href = "{}/api/vdc/{}/action/instantiateVAppTemplate".format(self.url,
1657                                                                                                 vdc_id)
1658                 data = """<?xml version="1.0" encoding="UTF-8"?>
1659                 <InstantiateVAppTemplateParams
1660                 xmlns="http://www.vmware.com/vcloud/v1.5"
1661                 name="{}"
1662                 deploy="false"
1663                 powerOn="false"
1664                 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
1665                 xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/1">
1666                 <Description>Vapp instantiation</Description>
1667                 <InstantiationParams>
1668                      <NetworkConfigSection>
1669                          <ovf:Info>Configuration parameters for logical networks</ovf:Info>
1670                          <NetworkConfig networkName="None">
1671                              <Configuration>
1672                                  <ParentNetwork href=""/>
1673                                  <FenceMode>bridged</FenceMode>
1674                              </Configuration>
1675                          </NetworkConfig>
1676                      </NetworkConfigSection>
1677                 <LeaseSettingsSection
1678                 type="application/vnd.vmware.vcloud.leaseSettingsSection+xml">
1679                 <ovf:Info>Lease Settings</ovf:Info>
1680                 <StorageLeaseInSeconds>172800</StorageLeaseInSeconds>
1681                 <StorageLeaseExpiration>2014-04-25T08:08:16.438-07:00</StorageLeaseExpiration>
1682                 </LeaseSettingsSection>
1683                 </InstantiationParams>
1684                 <Source href="{}"/>
1685                 <SourcedItem>
1686                 <Source href="{}" id="{}" name="{}"
1687                 type="application/vnd.vmware.vcloud.vm+xml"/>
1688                 <VmGeneralParams>
1689                     <NeedsCustomization>false</NeedsCustomization>
1690                 </VmGeneralParams>
1691                 <InstantiationParams>
1692                       <NetworkConnectionSection>
1693                       <ovf:Info>Specifies the available VM network connections</ovf:Info>
1694                       <NetworkConnection network="{}">
1695                       <NetworkConnectionIndex>0</NetworkConnectionIndex>
1696                       <IsConnected>true</IsConnected>
1697                       <IpAddressAllocationMode>DHCP</IpAddressAllocationMode>
1698                       </NetworkConnection>
1699                       </NetworkConnectionSection><ovf:VirtualHardwareSection>
1700                       <ovf:Info>Virtual hardware requirements</ovf:Info>
1701                       <ovf:Item xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData"
1702                       xmlns:vmw="http://www.vmware.com/schema/ovf">
1703                       <rasd:AllocationUnits>hertz * 10^6</rasd:AllocationUnits>
1704                       <rasd:Description>Number of Virtual CPUs</rasd:Description>
1705                       <rasd:ElementName xmlns:py="http://codespeak.net/lxml/objectify/pytype" py:pytype="str">{cpu} virtual CPU(s)</rasd:ElementName>
1706                       <rasd:InstanceID>4</rasd:InstanceID>
1707                       <rasd:Reservation>0</rasd:Reservation>
1708                       <rasd:ResourceType>3</rasd:ResourceType>
1709                       <rasd:VirtualQuantity xmlns:py="http://codespeak.net/lxml/objectify/pytype" py:pytype="int">{cpu}</rasd:VirtualQuantity>
1710                       <rasd:Weight>0</rasd:Weight>
1711                       <vmw:CoresPerSocket ovf:required="false">{core}</vmw:CoresPerSocket>
1712                       </ovf:Item><ovf:Item xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData">
1713                       <rasd:AllocationUnits>byte * 2^20</rasd:AllocationUnits>
1714                       <rasd:Description>Memory Size</rasd:Description>
1715                       <rasd:ElementName xmlns:py="http://codespeak.net/lxml/objectify/pytype" py:pytype="str">{memory} MB of memory</rasd:ElementName>
1716                       <rasd:InstanceID>5</rasd:InstanceID>
1717                       <rasd:Reservation>0</rasd:Reservation>
1718                       <rasd:ResourceType>4</rasd:ResourceType>
1719                       <rasd:VirtualQuantity xmlns:py="http://codespeak.net/lxml/objectify/pytype" py:pytype="int">{memory}</rasd:VirtualQuantity>
1720                       <rasd:Weight>0</rasd:Weight>
1721                       </ovf:Item>
1722                 </ovf:VirtualHardwareSection>
1723                 </InstantiationParams>
1724                 </SourcedItem>
1725                 <AllEULAsAccepted>false</AllEULAsAccepted>
1726                 </InstantiateVAppTemplateParams>""".format(vmname_andid,
1727                                                      vapp_tempalte_href,
1728                                                                 vm_href,
1729                                                                   vm_id,
1730                                                                 vm_name,
1731                                                         primary_netname,
1732                                                                cpu=cpus,
1733                                                              core=cores,
1734                                                        memory=memory_mb)
1735
1736                 response = self.perform_request(req_type='POST',
1737                                                 url=instantiate_vapp_href,
1738                                                 headers=headers,
1739                                                 data=data)
1740
1741                 if response.status_code != 201:
1742                     self.logger.error("REST call {} failed reason : {}"\
1743                          "status code : {}".format(instantiate_vapp_href,
1744                                                         response.content,
1745                                                    response.status_code))
1746                     raise vimconn.vimconnException("new_vminstance(): Failed to create"\
1747                                                         "vAapp {}".format(vmname_andid))
1748                 else:
1749                     vapptask = self.get_task_from_response(response.content)
1750
1751                 if vapptask is None and retry==1:
1752                     self.get_token() # Retry getting token
1753                     continue
1754                 else:
1755                     break
1756
1757             if vapptask is None or vapptask is False:
1758                 raise vimconn.vimconnUnexpectedResponse(
1759                     "new_vminstance(): failed to create vApp {}".format(vmname_andid))
1760
1761             # wait for task to complete
1762             result = self.client.get_task_monitor().wait_for_success(task=vapptask)
1763
1764             if result.get('status') == 'success':
1765                 self.logger.debug("new_vminstance(): Sucessfully created Vapp {}".format(vmname_andid))
1766             else:
1767                 raise vimconn.vimconnUnexpectedResponse(
1768                     "new_vminstance(): failed to create vApp {}".format(vmname_andid))
1769
1770         except Exception as exp:
1771             raise vimconn.vimconnUnexpectedResponse(
1772                 "new_vminstance(): failed to create vApp {} with Exception:{}".format(vmname_andid, exp))
1773
1774         # we should have now vapp in undeployed state.
1775         try:
1776             vdc_obj = VDC(self.client, href=vdc.get('href'))
1777             vapp_resource = vdc_obj.get_vapp(vmname_andid)
1778             vapp_uuid = vapp_resource.get('id').split(':')[-1]
1779             vapp = VApp(self.client, resource=vapp_resource)
1780
1781         except Exception as exp:
1782             raise vimconn.vimconnUnexpectedResponse(
1783                     "new_vminstance(): Failed to retrieve vApp {} after creation: Exception:{}"
1784                     .format(vmname_andid, exp))
1785
1786         if vapp_uuid is None:
1787             raise vimconn.vimconnUnexpectedResponse(
1788                 "new_vminstance(): Failed to retrieve vApp {} after creation".format(
1789                                                                             vmname_andid))
1790
1791         #Add PCI passthrough/SRIOV configrations
1792         vm_obj = None
1793         pci_devices_info = []
1794         sriov_net_info = []
1795         reserve_memory = False
1796
1797         for net in net_list:
1798             if net["type"] == "PF" or net["type"] == "PCI-PASSTHROUGH":
1799                 pci_devices_info.append(net)
1800             elif (net["type"] == "VF" or net["type"] == "SR-IOV" or net["type"] == "VFnotShared") and 'net_id'in net:
1801                 sriov_net_info.append(net)
1802
1803         #Add PCI
1804         if len(pci_devices_info) > 0:
1805             self.logger.info("Need to add PCI devices {} into VM {}".format(pci_devices_info,
1806                                                                         vmname_andid ))
1807             PCI_devices_status, vm_obj, vcenter_conect = self.add_pci_devices(vapp_uuid,
1808                                                                             pci_devices_info,
1809                                                                             vmname_andid)
1810             if PCI_devices_status:
1811                 self.logger.info("Added PCI devives {} to VM {}".format(
1812                                                             pci_devices_info,
1813                                                             vmname_andid)
1814                                  )
1815                 reserve_memory = True
1816             else:
1817                 self.logger.info("Fail to add PCI devives {} to VM {}".format(
1818                                                             pci_devices_info,
1819                                                             vmname_andid)
1820                                  )
1821
1822         # Modify vm disk
1823         if vm_disk:
1824             #Assuming there is only one disk in ovf and fast provisioning in organization vDC is disabled
1825             result = self.modify_vm_disk(vapp_uuid, vm_disk)
1826             if result :
1827                 self.logger.debug("Modified Disk size of VM {} ".format(vmname_andid))
1828
1829         #Add new or existing disks to vApp
1830         if disk_list:
1831             added_existing_disk = False
1832             for disk in disk_list:
1833                 if 'device_type' in disk and disk['device_type'] == 'cdrom':
1834                     image_id = disk['image_id']
1835                     # Adding CD-ROM to VM
1836                     # will revisit code once specification ready to support this feature
1837                     self.insert_media_to_vm(vapp, image_id)
1838                 elif "image_id" in disk and disk["image_id"] is not None:
1839                     self.logger.debug("Adding existing disk from image {} to vm {} ".format(
1840                                                                     disk["image_id"] , vapp_uuid))
1841                     self.add_existing_disk(catalogs=catalogs,
1842                                            image_id=disk["image_id"],
1843                                            size = disk["size"],
1844                                            template_name=templateName,
1845                                            vapp_uuid=vapp_uuid
1846                                            )
1847                     added_existing_disk = True
1848                 else:
1849                     #Wait till added existing disk gets reflected into vCD database/API
1850                     if added_existing_disk:
1851                         time.sleep(5)
1852                         added_existing_disk = False
1853                     self.add_new_disk(vapp_uuid, disk['size'])
1854
1855         if numas:
1856             # Assigning numa affinity setting
1857             for numa in numas:
1858                 if 'paired-threads-id' in numa:
1859                     paired_threads_id = numa['paired-threads-id']
1860                     self.set_numa_affinity(vapp_uuid, paired_threads_id)
1861
1862         # add NICs & connect to networks in netlist
1863         try:
1864             self.logger.info("Request to connect VM to a network: {}".format(net_list))
1865             nicIndex = 0
1866             primary_nic_index = 0
1867             for net in net_list:
1868                 # openmano uses network id in UUID format.
1869                 # vCloud Director need a name so we do reverse operation from provided UUID we lookup a name
1870                 # [{'use': 'bridge', 'net_id': '527d4bf7-566a-41e7-a9e7-ca3cdd9cef4f', 'type': 'virtual',
1871                 #   'vpci': '0000:00:11.0', 'name': 'eth0'}]
1872
1873                 if 'net_id' not in net:
1874                     continue
1875
1876                 #Using net_id as a vim_id i.e. vim interface id, as do not have saperate vim interface id
1877                 #Same will be returned in refresh_vms_status() as vim_interface_id
1878                 net['vim_id'] = net['net_id']  # Provide the same VIM identifier as the VIM network
1879
1880                 interface_net_id = net['net_id']
1881                 interface_net_name = self.get_network_name_by_id(network_uuid=interface_net_id)
1882                 interface_network_mode = net['use']
1883
1884                 if interface_network_mode == 'mgmt':
1885                     primary_nic_index = nicIndex
1886
1887                 """- POOL (A static IP address is allocated automatically from a pool of addresses.)
1888                                   - DHCP (The IP address is obtained from a DHCP service.)
1889                                   - MANUAL (The IP address is assigned manually in the IpAddress element.)
1890                                   - NONE (No IP addressing mode specified.)"""
1891
1892                 if primary_netname is not None:
1893                     nets = filter(lambda n: n.get('name') == interface_net_name, self.get_network_list())
1894                     #For python3
1895                     #nets = [n for n in self.get_network_list() if n.get('name') == interface_net_name]
1896                     if len(nets) == 1:
1897                         self.logger.info("new_vminstance(): Found requested network: {}".format(nets[0].get('name')))
1898
1899                         vdc_obj = VDC(self.client, href=vdc.get('href'))
1900                         vapp_resource = vdc_obj.get_vapp(vmname_andid)
1901                         vapp = VApp(self.client, resource=vapp_resource)
1902                         # connect network to VM - with all DHCP by default
1903                         task = vapp.connect_org_vdc_network(nets[0].get('name'))
1904
1905                         self.client.get_task_monitor().wait_for_success(task=task)
1906
1907                         type_list = ('PF', 'PCI-PASSTHROUGH', 'VF', 'SR-IOV', 'VFnotShared')
1908                         if 'type' in net and net['type'] not in type_list:
1909                             # fetching nic type from vnf
1910                             if 'model' in net:
1911                                 if net['model'] is not None and net['model'].lower() == 'virtio':
1912                                     nic_type = 'VMXNET3'
1913                                 else:
1914                                     nic_type = net['model']
1915
1916                                 self.logger.info("new_vminstance(): adding network adapter "\
1917                                                           "to a network {}".format(nets[0].get('name')))
1918                                 self.add_network_adapter_to_vms(vapp, nets[0].get('name'),
1919                                                                 primary_nic_index,
1920                                                                 nicIndex,
1921                                                                 net,
1922                                                                 nic_type=nic_type)
1923                             else:
1924                                 self.logger.info("new_vminstance(): adding network adapter "\
1925                                                          "to a network {}".format(nets[0].get('name')))
1926                                 self.add_network_adapter_to_vms(vapp, nets[0].get('name'),
1927                                                                 primary_nic_index,
1928                                                                 nicIndex,
1929                                                                 net)
1930                 nicIndex += 1
1931
1932             # cloud-init for ssh-key injection
1933             if cloud_config:
1934                 self.cloud_init(vapp,cloud_config)
1935
1936         # ############# Stub code for SRIOV #################
1937         #Add SRIOV
1938 #         if len(sriov_net_info) > 0:
1939 #             self.logger.info("Need to add SRIOV adapters {} into VM {}".format(sriov_net_info,
1940 #                                                                         vmname_andid ))
1941 #             sriov_status, vm_obj, vcenter_conect = self.add_sriov(vapp_uuid,
1942 #                                                                   sriov_net_info,
1943 #                                                                   vmname_andid)
1944 #             if sriov_status:
1945 #                 self.logger.info("Added SRIOV {} to VM {}".format(
1946 #                                                             sriov_net_info,
1947 #                                                             vmname_andid)
1948 #                                  )
1949 #                 reserve_memory = True
1950 #             else:
1951 #                 self.logger.info("Fail to add SRIOV {} to VM {}".format(
1952 #                                                             sriov_net_info,
1953 #                                                             vmname_andid)
1954 #                                  )
1955
1956             # If VM has PCI devices or SRIOV reserve memory for VM
1957             if reserve_memory:
1958                 memReserve = vm_obj.config.hardware.memoryMB
1959                 spec = vim.vm.ConfigSpec()
1960                 spec.memoryAllocation = vim.ResourceAllocationInfo(reservation=memReserve)
1961                 task = vm_obj.ReconfigVM_Task(spec=spec)
1962                 if task:
1963                     result = self.wait_for_vcenter_task(task, vcenter_conect)
1964                     self.logger.info("Reserved memory {} MB for "
1965                                      "VM VM status: {}".format(str(memReserve), result))
1966                 else:
1967                     self.logger.info("Fail to reserved memory {} to VM {}".format(
1968                                                                 str(memReserve), str(vm_obj)))
1969
1970             self.logger.debug("new_vminstance(): starting power on vApp {} ".format(vmname_andid))
1971
1972             vapp_id = vapp_resource.get('id').split(':')[-1]
1973             poweron_task = self.power_on_vapp(vapp_id, vmname_andid)
1974             result = self.client.get_task_monitor().wait_for_success(task=poweron_task)
1975             if result.get('status') == 'success':
1976                 self.logger.info("new_vminstance(): Successfully power on "\
1977                                              "vApp {}".format(vmname_andid))
1978             else:
1979                 self.logger.error("new_vminstance(): failed to power on vApp "\
1980                                                      "{}".format(vmname_andid))
1981
1982         except Exception as exp :
1983             # it might be a case if specific mandatory entry in dict is empty or some other pyVcloud exception
1984             self.logger.error("new_vminstance(): Failed create new vm instance {} with exception {}"
1985                               .format(name, exp))
1986             raise vimconn.vimconnException("new_vminstance(): Failed create new vm instance {} with exception {}"
1987                                            .format(name, exp))
1988
1989         # check if vApp deployed and if that the case return vApp UUID otherwise -1
1990         wait_time = 0
1991         vapp_uuid = None
1992         while wait_time <= MAX_WAIT_TIME:
1993             try:
1994                 vapp_resource = vdc_obj.get_vapp(vmname_andid)
1995                 vapp = VApp(self.client, resource=vapp_resource)
1996             except Exception as exp:
1997                 raise vimconn.vimconnUnexpectedResponse(
1998                         "new_vminstance(): Failed to retrieve vApp {} after creation: Exception:{}"
1999                         .format(vmname_andid, exp))
2000
2001             #if vapp and vapp.me.deployed:
2002             if vapp and vapp_resource.get('deployed') == 'true':
2003                 vapp_uuid = vapp_resource.get('id').split(':')[-1]
2004                 break
2005             else:
2006                 self.logger.debug("new_vminstance(): Wait for vApp {} to deploy".format(name))
2007                 time.sleep(INTERVAL_TIME)
2008
2009             wait_time +=INTERVAL_TIME
2010
2011         #SET Affinity Rule for VM
2012         #Pre-requisites: User has created Hosh Groups in vCenter with respective Hosts to be used
2013         #While creating VIM account user has to pass the Host Group names in availability_zone list
2014         #"availability_zone" is a  part of VIM "config" parameters
2015         #For example, in VIM config: "availability_zone":["HG_170","HG_174","HG_175"]
2016         #Host groups are referred as availability zones
2017         #With following procedure, deployed VM will be added into a VM group.
2018         #Then A VM to Host Affinity rule will be created using the VM group & Host group.
2019         if(availability_zone_list):
2020             self.logger.debug("Existing Host Groups in VIM {}".format(self.config.get('availability_zone')))
2021             #Admin access required for creating Affinity rules
2022             client = self.connect_as_admin()
2023             if not client:
2024                 raise vimconn.vimconnConnectionException("Failed to connect vCD as admin")
2025             else:
2026                 self.client = client
2027             if self.client:
2028                 headers = {'Accept':'application/*+xml;version=27.0',
2029                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
2030             #Step1: Get provider vdc details from organization
2031             pvdc_href = self.get_pvdc_for_org(self.tenant_name, headers)
2032             if pvdc_href is not None:
2033             #Step2: Found required pvdc, now get resource pool information
2034                 respool_href = self.get_resource_pool_details(pvdc_href, headers)
2035                 if respool_href is None:
2036                     #Raise error if respool_href not found
2037                     msg = "new_vminstance():Error in finding resource pool details in pvdc {}"\
2038                            .format(pvdc_href)
2039                     self.log_message(msg)
2040
2041             #Step3: Verify requested availability zone(hostGroup) is present in vCD
2042             # get availability Zone
2043             vm_az = self.get_vm_availability_zone(availability_zone_index, availability_zone_list)
2044             # check if provided av zone(hostGroup) is present in vCD VIM
2045             status = self.check_availibility_zone(vm_az, respool_href, headers)
2046             if status is False:
2047                 msg = "new_vminstance(): Error in finding availability zone(Host Group): {} in "\
2048                        "resource pool {} status: {}".format(vm_az,respool_href,status)
2049                 self.log_message(msg)
2050             else:
2051                 self.logger.debug ("new_vminstance(): Availability zone {} found in VIM".format(vm_az))
2052
2053             #Step4: Find VM group references to create vm group
2054             vmgrp_href = self.find_vmgroup_reference(respool_href, headers)
2055             if vmgrp_href == None:
2056                 msg = "new_vminstance(): No reference to VmGroup found in resource pool"
2057                 self.log_message(msg)
2058
2059             #Step5: Create a VmGroup with name az_VmGroup
2060             vmgrp_name = vm_az + "_" + name #Formed VM Group name = Host Group name + VM name
2061             status = self.create_vmgroup(vmgrp_name, vmgrp_href, headers)
2062             if status is not True:
2063                 msg = "new_vminstance(): Error in creating VM group {}".format(vmgrp_name)
2064                 self.log_message(msg)
2065
2066             #VM Group url to add vms to vm group
2067             vmgrpname_url = self.url + "/api/admin/extension/vmGroup/name/"+ vmgrp_name
2068
2069             #Step6: Add VM to VM Group
2070             #Find VM uuid from vapp_uuid
2071             vm_details = self.get_vapp_details_rest(vapp_uuid)
2072             vm_uuid = vm_details['vmuuid']
2073
2074             status = self.add_vm_to_vmgroup(vm_uuid, vmgrpname_url, vmgrp_name, headers)
2075             if status is not True:
2076                 msg = "new_vminstance(): Error in adding VM to VM group {}".format(vmgrp_name)
2077                 self.log_message(msg)
2078
2079             #Step7: Create VM to Host affinity rule
2080             addrule_href = self.get_add_rule_reference (respool_href, headers)
2081             if addrule_href is None:
2082                 msg = "new_vminstance(): Error in finding href to add rule in resource pool: {}"\
2083                       .format(respool_href)
2084                 self.log_message(msg)
2085
2086             status = self.create_vm_to_host_affinity_rule(addrule_href, vmgrp_name, vm_az, "Affinity",  headers)
2087             if status is False:
2088                 msg = "new_vminstance(): Error in creating affinity rule for VM {} in Host group {}"\
2089                       .format(name, vm_az)
2090                 self.log_message(msg)
2091             else:
2092                 self.logger.debug("new_vminstance(): Affinity rule created successfully. Added {} in Host group {}"\
2093                                     .format(name, vm_az))
2094             #Reset token to a normal user to perform other operations
2095             self.get_token()
2096
2097         if vapp_uuid is not None:
2098             return vapp_uuid, None
2099         else:
2100             raise vimconn.vimconnUnexpectedResponse("new_vminstance(): Failed create new vm instance {}".format(name))
2101
2102
2103     def get_vcd_availibility_zones(self,respool_href, headers):
2104         """ Method to find presence of av zone is VIM resource pool
2105
2106             Args:
2107                 respool_href - resource pool href
2108                 headers - header information
2109
2110             Returns:
2111                vcd_az - list of azone present in vCD
2112         """
2113         vcd_az = []
2114         url=respool_href
2115         resp = self.perform_request(req_type='GET',url=respool_href, headers=headers)
2116
2117         if resp.status_code != requests.codes.ok:
2118             self.logger.debug ("REST API call {} failed. Return status code {}".format(url, resp.status_code))
2119         else:
2120         #Get the href to hostGroups and find provided hostGroup is present in it
2121             resp_xml = XmlElementTree.fromstring(resp.content)
2122             for child in resp_xml:
2123                 if 'VMWProviderVdcResourcePool' in child.tag:
2124                     for schild in child:
2125                         if 'Link' in schild.tag:
2126                             if schild.attrib.get('type') == "application/vnd.vmware.admin.vmwHostGroupsType+xml":
2127                                 hostGroup = schild.attrib.get('href')
2128                                 hg_resp = self.perform_request(req_type='GET',url=hostGroup, headers=headers)
2129                                 if hg_resp.status_code != requests.codes.ok:
2130                                     self.logger.debug ("REST API call {} failed. Return status code {}".format(hostGroup, hg_resp.status_code))
2131                                 else:
2132                                     hg_resp_xml =  XmlElementTree.fromstring(hg_resp.content)
2133                                     for hostGroup in hg_resp_xml:
2134                                         if 'HostGroup' in hostGroup.tag:
2135                                             #append host group name to the list
2136                                             vcd_az.append(hostGroup.attrib.get("name"))
2137         return vcd_az
2138
2139
2140     def set_availability_zones(self):
2141         """
2142         Set vim availability zone
2143         """
2144
2145         vim_availability_zones = None
2146         availability_zone = None
2147         if 'availability_zone' in self.config:
2148             vim_availability_zones = self.config.get('availability_zone')
2149         if isinstance(vim_availability_zones, str):
2150             availability_zone = [vim_availability_zones]
2151         elif isinstance(vim_availability_zones, list):
2152             availability_zone = vim_availability_zones
2153         else:
2154             return availability_zone
2155
2156         return availability_zone
2157
2158
2159     def get_vm_availability_zone(self, availability_zone_index, availability_zone_list):
2160         """
2161         Return the availability zone to be used by the created VM.
2162         returns: The VIM availability zone to be used or None
2163         """
2164         if availability_zone_index is None:
2165             if not self.config.get('availability_zone'):
2166                 return None
2167             elif isinstance(self.config.get('availability_zone'), str):
2168                 return self.config['availability_zone']
2169             else:
2170                 return self.config['availability_zone'][0]
2171
2172         vim_availability_zones = self.availability_zone
2173
2174         # check if VIM offer enough availability zones describe in the VNFD
2175         if vim_availability_zones and len(availability_zone_list) <= len(vim_availability_zones):
2176             # check if all the names of NFV AV match VIM AV names
2177             match_by_index = False
2178             for av in availability_zone_list:
2179                 if av not in vim_availability_zones:
2180                     match_by_index = True
2181                     break
2182             if match_by_index:
2183                 self.logger.debug("Required Availability zone or Host Group not found in VIM config")
2184                 self.logger.debug("Input Availability zone list: {}".format(availability_zone_list))
2185                 self.logger.debug("VIM configured Availability zones: {}".format(vim_availability_zones))
2186                 self.logger.debug("VIM Availability zones will be used by index")
2187                 return vim_availability_zones[availability_zone_index]
2188             else:
2189                 return availability_zone_list[availability_zone_index]
2190         else:
2191             raise vimconn.vimconnConflictException("No enough availability zones at VIM for this deployment")
2192
2193
2194     def create_vm_to_host_affinity_rule(self, addrule_href, vmgrpname, hostgrpname, polarity, headers):
2195         """ Method to create VM to Host Affinity rule in vCD
2196
2197         Args:
2198             addrule_href - href to make a POST request
2199             vmgrpname - name of the VM group created
2200             hostgrpnmae - name of the host group created earlier
2201             polarity - Affinity or Anti-affinity (default: Affinity)
2202             headers - headers to make REST call
2203
2204         Returns:
2205             True- if rule is created
2206             False- Failed to create rule due to some error
2207
2208         """
2209         task_status = False
2210         rule_name = polarity + "_" + vmgrpname
2211         payload = """<?xml version="1.0" encoding="UTF-8"?>
2212                      <vmext:VMWVmHostAffinityRule
2213                        xmlns:vmext="http://www.vmware.com/vcloud/extension/v1.5"
2214                        xmlns:vcloud="http://www.vmware.com/vcloud/v1.5"
2215                        type="application/vnd.vmware.admin.vmwVmHostAffinityRule+xml">
2216                        <vcloud:Name>{}</vcloud:Name>
2217                        <vcloud:IsEnabled>true</vcloud:IsEnabled>
2218                        <vcloud:IsMandatory>true</vcloud:IsMandatory>
2219                        <vcloud:Polarity>{}</vcloud:Polarity>
2220                        <vmext:HostGroupName>{}</vmext:HostGroupName>
2221                        <vmext:VmGroupName>{}</vmext:VmGroupName>
2222                      </vmext:VMWVmHostAffinityRule>""".format(rule_name, polarity, hostgrpname, vmgrpname)
2223
2224         resp = self.perform_request(req_type='POST',url=addrule_href, headers=headers, data=payload)
2225
2226         if resp.status_code != requests.codes.accepted:
2227             self.logger.debug ("REST API call {} failed. Return status code {}".format(addrule_href, resp.status_code))
2228             task_status = False
2229             return task_status
2230         else:
2231             affinity_task = self.get_task_from_response(resp.content)
2232             self.logger.debug ("affinity_task: {}".format(affinity_task))
2233             if affinity_task is None or affinity_task is False:
2234                 raise vimconn.vimconnUnexpectedResponse("failed to find affinity task")
2235             # wait for task to complete
2236             result = self.client.get_task_monitor().wait_for_success(task=affinity_task)
2237             if result.get('status') == 'success':
2238                 self.logger.debug("Successfully created affinity rule {}".format(rule_name))
2239                 return True
2240             else:
2241                 raise vimconn.vimconnUnexpectedResponse(
2242                       "failed to create affinity rule {}".format(rule_name))
2243
2244
2245     def get_add_rule_reference (self, respool_href, headers):
2246         """ This method finds href to add vm to host affinity rule to vCD
2247
2248         Args:
2249             respool_href- href to resource pool
2250             headers- header information to make REST call
2251
2252         Returns:
2253             None - if no valid href to add rule found or
2254             addrule_href - href to add vm to host affinity rule of resource pool
2255         """
2256         addrule_href = None
2257         resp = self.perform_request(req_type='GET',url=respool_href, headers=headers)
2258
2259         if resp.status_code != requests.codes.ok:
2260             self.logger.debug ("REST API call {} failed. Return status code {}".format(respool_href, resp.status_code))
2261         else:
2262
2263             resp_xml = XmlElementTree.fromstring(resp.content)
2264             for child in resp_xml:
2265                 if 'VMWProviderVdcResourcePool' in child.tag:
2266                     for schild in child:
2267                         if 'Link' in schild.tag:
2268                             if schild.attrib.get('type') == "application/vnd.vmware.admin.vmwVmHostAffinityRule+xml" and \
2269                                 schild.attrib.get('rel') == "add":
2270                                 addrule_href = schild.attrib.get('href')
2271                                 break
2272
2273         return addrule_href
2274
2275
2276     def add_vm_to_vmgroup(self, vm_uuid, vmGroupNameURL, vmGroup_name, headers):
2277         """ Method to add deployed VM to newly created VM Group.
2278             This is required to create VM to Host affinity in vCD
2279
2280         Args:
2281             vm_uuid- newly created vm uuid
2282             vmGroupNameURL- URL to VM Group name
2283             vmGroup_name- Name of VM group created
2284             headers- Headers for REST request
2285
2286         Returns:
2287             True- if VM added to VM group successfully
2288             False- if any error encounter
2289         """
2290
2291         addvm_resp = self.perform_request(req_type='GET',url=vmGroupNameURL, headers=headers)#, data=payload)
2292
2293         if addvm_resp.status_code != requests.codes.ok:
2294             self.logger.debug ("REST API call to get VM Group Name url {} failed. Return status code {}"\
2295                                .format(vmGroupNameURL, addvm_resp.status_code))
2296             return False
2297         else:
2298             resp_xml = XmlElementTree.fromstring(addvm_resp.content)
2299             for child in resp_xml:
2300                 if child.tag.split('}')[1] == 'Link':
2301                     if child.attrib.get("rel") == "addVms":
2302                         addvmtogrpURL =  child.attrib.get("href")
2303
2304         #Get vm details
2305         url_list = [self.url, '/api/vApp/vm-',vm_uuid]
2306         vmdetailsURL = ''.join(url_list)
2307
2308         resp = self.perform_request(req_type='GET',url=vmdetailsURL, headers=headers)
2309
2310         if resp.status_code != requests.codes.ok:
2311             self.logger.debug ("REST API call {} failed. Return status code {}".format(vmdetailsURL, resp.status_code))
2312             return False
2313
2314         #Parse VM details
2315         resp_xml = XmlElementTree.fromstring(resp.content)
2316         if resp_xml.tag.split('}')[1] == "Vm":
2317             vm_id = resp_xml.attrib.get("id")
2318             vm_name = resp_xml.attrib.get("name")
2319             vm_href = resp_xml.attrib.get("href")
2320             #print vm_id, vm_name, vm_href
2321         #Add VM into VMgroup
2322         payload = """<?xml version="1.0" encoding="UTF-8"?>\
2323                    <ns2:Vms xmlns:ns2="http://www.vmware.com/vcloud/v1.5" \
2324                     xmlns="http://www.vmware.com/vcloud/versions" \
2325                     xmlns:ns3="http://schemas.dmtf.org/ovf/envelope/1" \
2326                     xmlns:ns4="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData" \
2327                     xmlns:ns5="http://schemas.dmtf.org/wbem/wscim/1/common" \
2328                     xmlns:ns6="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData" \
2329                     xmlns:ns7="http://www.vmware.com/schema/ovf" \
2330                     xmlns:ns8="http://schemas.dmtf.org/ovf/environment/1" \
2331                     xmlns:ns9="http://www.vmware.com/vcloud/extension/v1.5">\
2332                     <ns2:VmReference href="{}" id="{}" name="{}" \
2333                     type="application/vnd.vmware.vcloud.vm+xml" />\
2334                    </ns2:Vms>""".format(vm_href, vm_id, vm_name)
2335
2336         addvmtogrp_resp = self.perform_request(req_type='POST',url=addvmtogrpURL, headers=headers, data=payload)
2337
2338         if addvmtogrp_resp.status_code != requests.codes.accepted:
2339             self.logger.debug ("REST API call {} failed. Return status code {}".format(addvmtogrpURL, addvmtogrp_resp.status_code))
2340             return False
2341         else:
2342             self.logger.debug ("Done adding VM {} to VMgroup {}".format(vm_name, vmGroup_name))
2343             return True
2344
2345
2346     def create_vmgroup(self, vmgroup_name, vmgroup_href, headers):
2347         """Method to create a VM group in vCD
2348
2349            Args:
2350               vmgroup_name : Name of VM group to be created
2351               vmgroup_href : href for vmgroup
2352               headers- Headers for REST request
2353         """
2354         #POST to add URL with required data
2355         vmgroup_status = False
2356         payload = """<VMWVmGroup xmlns="http://www.vmware.com/vcloud/extension/v1.5" \
2357                        xmlns:vcloud_v1.5="http://www.vmware.com/vcloud/v1.5" name="{}">\
2358                    <vmCount>1</vmCount>\
2359                    </VMWVmGroup>""".format(vmgroup_name)
2360         resp = self.perform_request(req_type='POST',url=vmgroup_href, headers=headers, data=payload)
2361
2362         if resp.status_code != requests.codes.accepted:
2363             self.logger.debug ("REST API call {} failed. Return status code {}".format(vmgroup_href, resp.status_code))
2364             return vmgroup_status
2365         else:
2366             vmgroup_task = self.get_task_from_response(resp.content)
2367             if vmgroup_task is None or vmgroup_task is False:
2368                 raise vimconn.vimconnUnexpectedResponse(
2369                     "create_vmgroup(): failed to create VM group {}".format(vmgroup_name))
2370
2371             # wait for task to complete
2372             result = self.client.get_task_monitor().wait_for_success(task=vmgroup_task)
2373
2374             if result.get('status') == 'success':
2375                 self.logger.debug("create_vmgroup(): Successfully created VM group {}".format(vmgroup_name))
2376                 #time.sleep(10)
2377                 vmgroup_status = True
2378                 return vmgroup_status
2379             else:
2380                 raise vimconn.vimconnUnexpectedResponse(\
2381                         "create_vmgroup(): failed to create VM group {}".format(vmgroup_name))
2382
2383
2384     def find_vmgroup_reference(self, url, headers):
2385         """ Method to create a new VMGroup which is required to add created VM
2386             Args:
2387                url- resource pool href
2388                headers- header information
2389
2390             Returns:
2391                returns href to VM group to create VM group
2392         """
2393         #Perform GET on resource pool to find 'add' link to create VMGroup
2394         #https://vcd-ip/api/admin/extension/providervdc/<providervdc id>/resourcePools
2395         vmgrp_href = None
2396         resp = self.perform_request(req_type='GET',url=url, headers=headers)
2397
2398         if resp.status_code != requests.codes.ok:
2399             self.logger.debug ("REST API call {} failed. Return status code {}".format(url, resp.status_code))
2400         else:
2401             #Get the href to add vmGroup to vCD
2402             resp_xml = XmlElementTree.fromstring(resp.content)
2403             for child in resp_xml:
2404                 if 'VMWProviderVdcResourcePool' in child.tag:
2405                     for schild in child:
2406                         if 'Link' in schild.tag:
2407                             #Find href with type VMGroup and rel with add
2408                             if schild.attrib.get('type') == "application/vnd.vmware.admin.vmwVmGroupType+xml"\
2409                                 and schild.attrib.get('rel') == "add":
2410                                 vmgrp_href = schild.attrib.get('href')
2411                                 return vmgrp_href
2412
2413
2414     def check_availibility_zone(self, az, respool_href, headers):
2415         """ Method to verify requested av zone is present or not in provided
2416             resource pool
2417
2418             Args:
2419                 az - name of hostgroup (availibility_zone)
2420                 respool_href - Resource Pool href
2421                 headers - Headers to make REST call
2422             Returns:
2423                 az_found - True if availibility_zone is found else False
2424         """
2425         az_found = False
2426         headers['Accept']='application/*+xml;version=27.0'
2427         resp = self.perform_request(req_type='GET',url=respool_href, headers=headers)
2428
2429         if resp.status_code != requests.codes.ok:
2430             self.logger.debug ("REST API call {} failed. Return status code {}".format(respool_href, resp.status_code))
2431         else:
2432         #Get the href to hostGroups and find provided hostGroup is present in it
2433             resp_xml = XmlElementTree.fromstring(resp.content)
2434
2435             for child in resp_xml:
2436                 if 'VMWProviderVdcResourcePool' in child.tag:
2437                     for schild in child:
2438                         if 'Link' in schild.tag:
2439                             if schild.attrib.get('type') == "application/vnd.vmware.admin.vmwHostGroupsType+xml":
2440                                 hostGroup_href = schild.attrib.get('href')
2441                                 hg_resp = self.perform_request(req_type='GET',url=hostGroup_href, headers=headers)
2442                                 if hg_resp.status_code != requests.codes.ok:
2443                                     self.logger.debug ("REST API call {} failed. Return status code {}".format(hostGroup_href, hg_resp.status_code))
2444                                 else:
2445                                     hg_resp_xml = XmlElementTree.fromstring(hg_resp.content)
2446                                     for hostGroup in hg_resp_xml:
2447                                         if 'HostGroup' in hostGroup.tag:
2448                                             if hostGroup.attrib.get("name") == az:
2449                                                 az_found = True
2450                                                 break
2451         return az_found
2452
2453
2454     def get_pvdc_for_org(self, org_vdc, headers):
2455         """ This method gets provider vdc references from organisation
2456
2457             Args:
2458                org_vdc - name of the organisation VDC to find pvdc
2459                headers - headers to make REST call
2460
2461             Returns:
2462                None - if no pvdc href found else
2463                pvdc_href - href to pvdc
2464         """
2465
2466         #Get provider VDC references from vCD
2467         pvdc_href = None
2468         #url = '<vcd url>/api/admin/extension/providerVdcReferences'
2469         url_list = [self.url, '/api/admin/extension/providerVdcReferences']
2470         url = ''.join(url_list)
2471
2472         response = self.perform_request(req_type='GET',url=url, headers=headers)
2473         if response.status_code != requests.codes.ok:
2474             self.logger.debug ("REST API call {} failed. Return status code {}"\
2475                                .format(url, response.status_code))
2476         else:
2477             xmlroot_response = XmlElementTree.fromstring(response.content)
2478             for child in xmlroot_response:
2479                 if 'ProviderVdcReference' in child.tag:
2480                     pvdc_href = child.attrib.get('href')
2481                     #Get vdcReferences to find org
2482                     pvdc_resp = self.perform_request(req_type='GET',url=pvdc_href, headers=headers)
2483                     if pvdc_resp.status_code != requests.codes.ok:
2484                         raise vimconn.vimconnException("REST API call {} failed. "\
2485                                                        "Return status code {}"\
2486                                                        .format(url, pvdc_resp.status_code))
2487
2488                     pvdc_resp_xml = XmlElementTree.fromstring(pvdc_resp.content)
2489                     for child in pvdc_resp_xml:
2490                         if 'Link' in child.tag:
2491                             if child.attrib.get('type') == "application/vnd.vmware.admin.vdcReferences+xml":
2492                                 vdc_href = child.attrib.get('href')
2493
2494                                 #Check if provided org is present in vdc
2495                                 vdc_resp = self.perform_request(req_type='GET',
2496                                                                 url=vdc_href,
2497                                                                 headers=headers)
2498                                 if vdc_resp.status_code != requests.codes.ok:
2499                                     raise vimconn.vimconnException("REST API call {} failed. "\
2500                                                                    "Return status code {}"\
2501                                                                    .format(url, vdc_resp.status_code))
2502                                 vdc_resp_xml = XmlElementTree.fromstring(vdc_resp.content)
2503                                 for child in vdc_resp_xml:
2504                                     if 'VdcReference' in child.tag:
2505                                         if child.attrib.get('name') == org_vdc:
2506                                             return pvdc_href
2507
2508
2509     def get_resource_pool_details(self, pvdc_href, headers):
2510         """ Method to get resource pool information.
2511             Host groups are property of resource group.
2512             To get host groups, we need to GET details of resource pool.
2513
2514             Args:
2515                 pvdc_href: href to pvdc details
2516                 headers: headers
2517
2518             Returns:
2519                 respool_href - Returns href link reference to resource pool
2520         """
2521         respool_href = None
2522         resp = self.perform_request(req_type='GET',url=pvdc_href, headers=headers)
2523
2524         if resp.status_code != requests.codes.ok:
2525             self.logger.debug ("REST API call {} failed. Return status code {}"\
2526                                .format(pvdc_href, resp.status_code))
2527         else:
2528             respool_resp_xml = XmlElementTree.fromstring(resp.content)
2529             for child in respool_resp_xml:
2530                 if 'Link' in child.tag:
2531                     if child.attrib.get('type') == "application/vnd.vmware.admin.vmwProviderVdcResourcePoolSet+xml":
2532                         respool_href = child.attrib.get("href")
2533                         break
2534         return respool_href
2535
2536
2537     def log_message(self, msg):
2538         """
2539             Method to log error messages related to Affinity rule creation
2540             in new_vminstance & raise Exception
2541                 Args :
2542                     msg - Error message to be logged
2543
2544         """
2545         #get token to connect vCD as a normal user
2546         self.get_token()
2547         self.logger.debug(msg)
2548         raise vimconn.vimconnException(msg)
2549
2550
2551     ##
2552     ##
2553     ##  based on current discussion
2554     ##
2555     ##
2556     ##  server:
2557     #   created: '2016-09-08T11:51:58'
2558     #   description: simple-instance.linux1.1
2559     #   flavor: ddc6776e-75a9-11e6-ad5f-0800273e724c
2560     #   hostId: e836c036-74e7-11e6-b249-0800273e724c
2561     #   image: dde30fe6-75a9-11e6-ad5f-0800273e724c
2562     #   status: ACTIVE
2563     #   error_msg:
2564     #   interfaces: …
2565     #
2566     def get_vminstance(self, vim_vm_uuid=None):
2567         """Returns the VM instance information from VIM"""
2568
2569         self.logger.debug("Client requesting vm instance {} ".format(vim_vm_uuid))
2570
2571         org, vdc = self.get_vdc_details()
2572         if vdc is None:
2573             raise vimconn.vimconnConnectionException(
2574                 "Failed to get a reference of VDC for a tenant {}".format(self.tenant_name))
2575
2576         vm_info_dict = self.get_vapp_details_rest(vapp_uuid=vim_vm_uuid)
2577         if not vm_info_dict:
2578             self.logger.debug("get_vminstance(): Failed to get vApp name by UUID {}".format(vim_vm_uuid))
2579             raise vimconn.vimconnNotFoundException("Failed to get vApp name by UUID {}".format(vim_vm_uuid))
2580
2581         status_key = vm_info_dict['status']
2582         error = ''
2583         try:
2584             vm_dict = {'created': vm_info_dict['created'],
2585                        'description': vm_info_dict['name'],
2586                        'status': vcdStatusCode2manoFormat[int(status_key)],
2587                        'hostId': vm_info_dict['vmuuid'],
2588                        'error_msg': error,
2589                        'vim_info': yaml.safe_dump(vm_info_dict), 'interfaces': []}
2590
2591             if 'interfaces' in vm_info_dict:
2592                 vm_dict['interfaces'] = vm_info_dict['interfaces']
2593             else:
2594                 vm_dict['interfaces'] = []
2595         except KeyError:
2596             vm_dict = {'created': '',
2597                        'description': '',
2598                        'status': vcdStatusCode2manoFormat[int(-1)],
2599                        'hostId': vm_info_dict['vmuuid'],
2600                        'error_msg': "Inconsistency state",
2601                        'vim_info': yaml.safe_dump(vm_info_dict), 'interfaces': []}
2602
2603         return vm_dict
2604
2605     def delete_vminstance(self, vm__vim_uuid, created_items=None):
2606         """Method poweroff and remove VM instance from vcloud director network.
2607
2608         Args:
2609             vm__vim_uuid: VM UUID
2610
2611         Returns:
2612             Returns the instance identifier
2613         """
2614
2615         self.logger.debug("Client requesting delete vm instance {} ".format(vm__vim_uuid))
2616
2617         org, vdc = self.get_vdc_details()
2618         vdc_obj = VDC(self.client, href=vdc.get('href'))
2619         if vdc_obj is None:
2620             self.logger.debug("delete_vminstance(): Failed to get a reference of VDC for a tenant {}".format(
2621                 self.tenant_name))
2622             raise vimconn.vimconnException(
2623                 "delete_vminstance(): Failed to get a reference of VDC for a tenant {}".format(self.tenant_name))
2624
2625         try:
2626             vapp_name = self.get_namebyvappid(vm__vim_uuid)
2627             vapp_resource = vdc_obj.get_vapp(vapp_name)
2628             vapp = VApp(self.client, resource=vapp_resource)
2629             if vapp_name is None:
2630                 self.logger.debug("delete_vminstance(): Failed to get vm by given {} vm uuid".format(vm__vim_uuid))
2631                 return -1, "delete_vminstance(): Failed to get vm by given {} vm uuid".format(vm__vim_uuid)
2632             else:
2633                 self.logger.info("Deleting vApp {} and UUID {}".format(vapp_name, vm__vim_uuid))
2634
2635             # Delete vApp and wait for status change if task executed and vApp is None.
2636
2637             if vapp:
2638                 if vapp_resource.get('deployed') == 'true':
2639                     self.logger.info("Powering off vApp {}".format(vapp_name))
2640                     #Power off vApp
2641                     powered_off = False
2642                     wait_time = 0
2643                     while wait_time <= MAX_WAIT_TIME:
2644                         power_off_task = vapp.power_off()
2645                         result = self.client.get_task_monitor().wait_for_success(task=power_off_task)
2646
2647                         if result.get('status') == 'success':
2648                             powered_off = True
2649                             break
2650                         else:
2651                             self.logger.info("Wait for vApp {} to power off".format(vapp_name))
2652                             time.sleep(INTERVAL_TIME)
2653
2654                         wait_time +=INTERVAL_TIME
2655                     if not powered_off:
2656                         self.logger.debug("delete_vminstance(): Failed to power off VM instance {} ".format(vm__vim_uuid))
2657                     else:
2658                         self.logger.info("delete_vminstance(): Powered off VM instance {} ".format(vm__vim_uuid))
2659
2660                     #Undeploy vApp
2661                     self.logger.info("Undeploy vApp {}".format(vapp_name))
2662                     wait_time = 0
2663                     undeployed = False
2664                     while wait_time <= MAX_WAIT_TIME:
2665                         vapp = VApp(self.client, resource=vapp_resource)
2666                         if not vapp:
2667                             self.logger.debug("delete_vminstance(): Failed to get vm by given {} vm uuid".format(vm__vim_uuid))
2668                             return -1, "delete_vminstance(): Failed to get vm by given {} vm uuid".format(vm__vim_uuid)
2669                         undeploy_task = vapp.undeploy()
2670
2671                         result = self.client.get_task_monitor().wait_for_success(task=undeploy_task)
2672                         if result.get('status') == 'success':
2673                             undeployed = True
2674                             break
2675                         else:
2676                             self.logger.debug("Wait for vApp {} to undeploy".format(vapp_name))
2677                             time.sleep(INTERVAL_TIME)
2678
2679                         wait_time +=INTERVAL_TIME
2680
2681                     if not undeployed:
2682                         self.logger.debug("delete_vminstance(): Failed to undeploy vApp {} ".format(vm__vim_uuid))
2683
2684                 # delete vapp
2685                 self.logger.info("Start deletion of vApp {} ".format(vapp_name))
2686
2687                 if vapp is not None:
2688                     wait_time = 0
2689                     result = False
2690
2691                     while wait_time <= MAX_WAIT_TIME:
2692                         vapp = VApp(self.client, resource=vapp_resource)
2693                         if not vapp:
2694                             self.logger.debug("delete_vminstance(): Failed to get vm by given {} vm uuid".format(vm__vim_uuid))
2695                             return -1, "delete_vminstance(): Failed to get vm by given {} vm uuid".format(vm__vim_uuid)
2696
2697                         delete_task = vdc_obj.delete_vapp(vapp.name, force=True)
2698
2699                         result = self.client.get_task_monitor().wait_for_success(task=delete_task)
2700                         if result.get('status') == 'success':
2701                             break
2702                         else:
2703                             self.logger.debug("Wait for vApp {} to delete".format(vapp_name))
2704                             time.sleep(INTERVAL_TIME)
2705
2706                         wait_time +=INTERVAL_TIME
2707
2708                     if result is None:
2709                         self.logger.debug("delete_vminstance(): Failed delete uuid {} ".format(vm__vim_uuid))
2710                     else:
2711                         self.logger.info("Deleted vm instance {} sccessfully".format(vm__vim_uuid))
2712                         return vm__vim_uuid
2713         except:
2714             self.logger.debug(traceback.format_exc())
2715             raise vimconn.vimconnException("delete_vminstance(): Failed delete vm instance {}".format(vm__vim_uuid))
2716
2717
2718     def refresh_vms_status(self, vm_list):
2719         """Get the status of the virtual machines and their interfaces/ports
2720            Params: the list of VM identifiers
2721            Returns a dictionary with:
2722                 vm_id:          #VIM id of this Virtual Machine
2723                     status:     #Mandatory. Text with one of:
2724                                 #  DELETED (not found at vim)
2725                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
2726                                 #  OTHER (Vim reported other status not understood)
2727                                 #  ERROR (VIM indicates an ERROR status)
2728                                 #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
2729                                 #  CREATING (on building process), ERROR
2730                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
2731                                 #
2732                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
2733                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
2734                     interfaces:
2735                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
2736                         mac_address:      #Text format XX:XX:XX:XX:XX:XX
2737                         vim_net_id:       #network id where this interface is connected
2738                         vim_interface_id: #interface/port VIM id
2739                         ip_address:       #null, or text with IPv4, IPv6 address
2740         """
2741
2742         self.logger.debug("Client requesting refresh vm status for {} ".format(vm_list))
2743
2744         org,vdc = self.get_vdc_details()
2745         if vdc is None:
2746             raise vimconn.vimconnException("Failed to get a reference of VDC for a tenant {}".format(self.tenant_name))
2747
2748         vms_dict = {}
2749         nsx_edge_list = []
2750         for vmuuid in vm_list:
2751             vapp_name = self.get_namebyvappid(vmuuid)
2752             if vapp_name is not None:
2753
2754                 try:
2755                     vm_pci_details = self.get_vm_pci_details(vmuuid)
2756                     vdc_obj = VDC(self.client, href=vdc.get('href'))
2757                     vapp_resource = vdc_obj.get_vapp(vapp_name)
2758                     the_vapp = VApp(self.client, resource=vapp_resource)
2759
2760                     vm_details = {}
2761                     for vm in the_vapp.get_all_vms():
2762                         headers = {'Accept':'application/*+xml;version=' + API_VERSION,
2763                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
2764                         response = self.perform_request(req_type='GET',
2765                                                         url=vm.get('href'),
2766                                                         headers=headers)
2767
2768                         if response.status_code != 200:
2769                             self.logger.error("refresh_vms_status : REST call {} failed reason : {}"\
2770                                                             "status code : {}".format(vm.get('href'),
2771                                                                                     response.content,
2772                                                                                response.status_code))
2773                             raise vimconn.vimconnException("refresh_vms_status : Failed to get "\
2774                                                                          "VM details")
2775                         xmlroot = XmlElementTree.fromstring(response.content)
2776
2777                         result = response.content.replace("\n"," ")
2778                         hdd_mb = re.search('vcloud:capacity="(\d+)"\svcloud:storageProfileOverrideVmDefault=',result).group(1)
2779                         vm_details['hdd_mb'] = int(hdd_mb) if hdd_mb else None
2780                         cpus = re.search('<rasd:Description>Number of Virtual CPUs</.*?>(\d+)</rasd:VirtualQuantity>',result).group(1)
2781                         vm_details['cpus'] = int(cpus) if cpus else None
2782                         memory_mb = re.search('<rasd:Description>Memory Size</.*?>(\d+)</rasd:VirtualQuantity>',result).group(1)
2783                         vm_details['memory_mb'] = int(memory_mb) if memory_mb else None
2784                         vm_details['status'] = vcdStatusCode2manoFormat[int(xmlroot.get('status'))]
2785                         vm_details['id'] = xmlroot.get('id')
2786                         vm_details['name'] = xmlroot.get('name')
2787                         vm_info = [vm_details]
2788                         if vm_pci_details:
2789                             vm_info[0].update(vm_pci_details)
2790
2791                         vm_dict = {'status': vcdStatusCode2manoFormat[int(vapp_resource.get('status'))],
2792                                    'error_msg': vcdStatusCode2manoFormat[int(vapp_resource.get('status'))],
2793                                    'vim_info': yaml.safe_dump(vm_info), 'interfaces': []}
2794
2795                         # get networks
2796                         vm_ip = None
2797                         vm_mac = None
2798                         networks = re.findall('<NetworkConnection needsCustomization=.*?</NetworkConnection>',result)
2799                         for network in networks:
2800                             mac_s = re.search('<MACAddress>(.*?)</MACAddress>',network)
2801                             vm_mac = mac_s.group(1) if mac_s else None
2802                             ip_s = re.search('<IpAddress>(.*?)</IpAddress>',network)
2803                             vm_ip = ip_s.group(1) if ip_s else None
2804
2805                             if vm_ip is None:
2806                                 if not nsx_edge_list:
2807                                     nsx_edge_list = self.get_edge_details()
2808                                     if nsx_edge_list is None:
2809                                         raise vimconn.vimconnException("refresh_vms_status:"\
2810                                                                        "Failed to get edge details from NSX Manager")
2811                                 if vm_mac is not None:
2812                                     vm_ip = self.get_ipaddr_from_NSXedge(nsx_edge_list, vm_mac)
2813
2814                             net_s = re.search('network="(.*?)"',network)
2815                             network_name = net_s.group(1) if net_s else None
2816
2817                             vm_net_id = self.get_network_id_by_name(network_name)
2818                             interface = {"mac_address": vm_mac,
2819                                          "vim_net_id": vm_net_id,
2820                                          "vim_interface_id": vm_net_id,
2821                                          "ip_address": vm_ip}
2822
2823                             vm_dict["interfaces"].append(interface)
2824
2825                     # add a vm to vm dict
2826                     vms_dict.setdefault(vmuuid, vm_dict)
2827                     self.logger.debug("refresh_vms_status : vm info {}".format(vm_dict))
2828                 except Exception as exp:
2829                     self.logger.debug("Error in response {}".format(exp))
2830                     self.logger.debug(traceback.format_exc())
2831
2832         return vms_dict
2833
2834
2835     def get_edge_details(self):
2836         """Get the NSX edge list from NSX Manager
2837            Returns list of NSX edges
2838         """
2839         edge_list = []
2840         rheaders = {'Content-Type': 'application/xml'}
2841         nsx_api_url = '/api/4.0/edges'
2842
2843         self.logger.debug("Get edge details from NSX Manager {} {}".format(self.nsx_manager, nsx_api_url))
2844
2845         try:
2846             resp = requests.get(self.nsx_manager + nsx_api_url,
2847                                 auth = (self.nsx_user, self.nsx_password),
2848                                 verify = False, headers = rheaders)
2849             if resp.status_code == requests.codes.ok:
2850                 paged_Edge_List = XmlElementTree.fromstring(resp.text)
2851                 for edge_pages in paged_Edge_List:
2852                     if edge_pages.tag == 'edgePage':
2853                         for edge_summary in edge_pages:
2854                             if edge_summary.tag == 'pagingInfo':
2855                                 for element in edge_summary:
2856                                     if element.tag == 'totalCount' and element.text == '0':
2857                                         raise vimconn.vimconnException("get_edge_details: No NSX edges details found: {}"
2858                                                                        .format(self.nsx_manager))
2859
2860                             if edge_summary.tag == 'edgeSummary':
2861                                 for element in edge_summary:
2862                                     if element.tag == 'id':
2863                                         edge_list.append(element.text)
2864                     else:
2865                         raise vimconn.vimconnException("get_edge_details: No NSX edge details found: {}"
2866                                                        .format(self.nsx_manager))
2867
2868                 if not edge_list:
2869                     raise vimconn.vimconnException("get_edge_details: "\
2870                                                    "No NSX edge details found: {}"
2871                                                    .format(self.nsx_manager))
2872                 else:
2873                     self.logger.debug("get_edge_details: Found NSX edges {}".format(edge_list))
2874                     return edge_list
2875             else:
2876                 self.logger.debug("get_edge_details: "
2877                                   "Failed to get NSX edge details from NSX Manager: {}"
2878                                   .format(resp.content))
2879                 return None
2880
2881         except Exception as exp:
2882             self.logger.debug("get_edge_details: "\
2883                               "Failed to get NSX edge details from NSX Manager: {}"
2884                               .format(exp))
2885             raise vimconn.vimconnException("get_edge_details: "\
2886                                            "Failed to get NSX edge details from NSX Manager: {}"
2887                                            .format(exp))
2888
2889
2890     def get_ipaddr_from_NSXedge(self, nsx_edges, mac_address):
2891         """Get IP address details from NSX edges, using the MAC address
2892            PARAMS: nsx_edges : List of NSX edges
2893                    mac_address : Find IP address corresponding to this MAC address
2894            Returns: IP address corrresponding to the provided MAC address
2895         """
2896
2897         ip_addr = None
2898         rheaders = {'Content-Type': 'application/xml'}
2899
2900         self.logger.debug("get_ipaddr_from_NSXedge: Finding IP addr from NSX edge")
2901
2902         try:
2903             for edge in nsx_edges:
2904                 nsx_api_url = '/api/4.0/edges/'+ edge +'/dhcp/leaseInfo'
2905
2906                 resp = requests.get(self.nsx_manager + nsx_api_url,
2907                                     auth = (self.nsx_user, self.nsx_password),
2908                                     verify = False, headers = rheaders)
2909
2910                 if resp.status_code == requests.codes.ok:
2911                     dhcp_leases = XmlElementTree.fromstring(resp.text)
2912                     for child in dhcp_leases:
2913                         if child.tag == 'dhcpLeaseInfo':
2914                             dhcpLeaseInfo = child
2915                             for leaseInfo in dhcpLeaseInfo:
2916                                 for elem in leaseInfo:
2917                                     if (elem.tag)=='macAddress':
2918                                         edge_mac_addr = elem.text
2919                                     if (elem.tag)=='ipAddress':
2920                                         ip_addr = elem.text
2921                                 if edge_mac_addr is not None:
2922                                     if edge_mac_addr == mac_address:
2923                                         self.logger.debug("Found ip addr {} for mac {} at NSX edge {}"
2924                                                           .format(ip_addr, mac_address,edge))
2925                                         return ip_addr
2926                 else:
2927                     self.logger.debug("get_ipaddr_from_NSXedge: "\
2928                                       "Error occurred while getting DHCP lease info from NSX Manager: {}"
2929                                       .format(resp.content))
2930
2931             self.logger.debug("get_ipaddr_from_NSXedge: No IP addr found in any NSX edge")
2932             return None
2933
2934         except XmlElementTree.ParseError as Err:
2935             self.logger.debug("ParseError in response from NSX Manager {}".format(Err.message), exc_info=True)
2936
2937
2938     def action_vminstance(self, vm__vim_uuid=None, action_dict=None, created_items={}):
2939         """Send and action over a VM instance from VIM
2940         Returns the vm_id if the action was successfully sent to the VIM"""
2941
2942         self.logger.debug("Received action for vm {} and action dict {}".format(vm__vim_uuid, action_dict))
2943         if vm__vim_uuid is None or action_dict is None:
2944             raise vimconn.vimconnException("Invalid request. VM id or action is None.")
2945
2946         org, vdc = self.get_vdc_details()
2947         if vdc is None:
2948             raise  vimconn.vimconnException("Failed to get a reference of VDC for a tenant {}".format(self.tenant_name))
2949
2950         vapp_name = self.get_namebyvappid(vm__vim_uuid)
2951         if vapp_name is None:
2952             self.logger.debug("action_vminstance(): Failed to get vm by given {} vm uuid".format(vm__vim_uuid))
2953             raise vimconn.vimconnException("Failed to get vm by given {} vm uuid".format(vm__vim_uuid))
2954         else:
2955             self.logger.info("Action_vminstance vApp {} and UUID {}".format(vapp_name, vm__vim_uuid))
2956
2957         try:
2958             vdc_obj = VDC(self.client, href=vdc.get('href'))
2959             vapp_resource = vdc_obj.get_vapp(vapp_name)
2960             vapp = VApp(self.client, resource=vapp_resource)
2961             if "start" in action_dict:
2962                 self.logger.info("action_vminstance: Power on vApp: {}".format(vapp_name))
2963                 poweron_task = self.power_on_vapp(vm__vim_uuid, vapp_name)
2964                 result = self.client.get_task_monitor().wait_for_success(task=poweron_task)
2965                 self.instance_actions_result("start", result, vapp_name)
2966             elif "rebuild" in action_dict:
2967                 self.logger.info("action_vminstance: Rebuild vApp: {}".format(vapp_name))
2968                 rebuild_task = vapp.deploy(power_on=True)
2969                 result = self.client.get_task_monitor().wait_for_success(task=rebuild_task)
2970                 self.instance_actions_result("rebuild", result, vapp_name)
2971             elif "pause" in action_dict:
2972                 self.logger.info("action_vminstance: pause vApp: {}".format(vapp_name))
2973                 pause_task = vapp.undeploy(action='suspend')
2974                 result = self.client.get_task_monitor().wait_for_success(task=pause_task)
2975                 self.instance_actions_result("pause", result, vapp_name)
2976             elif "resume" in action_dict:
2977                 self.logger.info("action_vminstance: resume vApp: {}".format(vapp_name))
2978                 poweron_task = self.power_on_vapp(vm__vim_uuid, vapp_name)
2979                 result = self.client.get_task_monitor().wait_for_success(task=poweron_task)
2980                 self.instance_actions_result("resume", result, vapp_name)
2981             elif "shutoff" in action_dict or "shutdown" in action_dict:
2982                 action_name , value = action_dict.items()[0]
2983                 #For python3
2984                 #action_name , value = list(action_dict.items())[0]
2985                 self.logger.info("action_vminstance: {} vApp: {}".format(action_name, vapp_name))
2986                 shutdown_task = vapp.shutdown()
2987                 result = self.client.get_task_monitor().wait_for_success(task=shutdown_task)
2988                 if action_name == "shutdown":
2989                     self.instance_actions_result("shutdown", result, vapp_name)
2990                 else:
2991                     self.instance_actions_result("shutoff", result, vapp_name)
2992             elif "forceOff" in action_dict:
2993                 result = vapp.undeploy(action='powerOff')
2994                 self.instance_actions_result("forceOff", result, vapp_name)
2995             elif "reboot" in action_dict:
2996                 self.logger.info("action_vminstance: reboot vApp: {}".format(vapp_name))
2997                 reboot_task = vapp.reboot()
2998                 self.client.get_task_monitor().wait_for_success(task=reboot_task)
2999             else:
3000                 raise vimconn.vimconnException("action_vminstance: Invalid action {} or action is None.".format(action_dict))
3001             return vm__vim_uuid
3002         except Exception as exp :
3003             self.logger.debug("action_vminstance: Failed with Exception {}".format(exp))
3004             raise vimconn.vimconnException("action_vminstance: Failed with Exception {}".format(exp))
3005
3006     def instance_actions_result(self, action, result, vapp_name):
3007         if result.get('status') == 'success':
3008             self.logger.info("action_vminstance: Sucessfully {} the vApp: {}".format(action, vapp_name))
3009         else:
3010             self.logger.error("action_vminstance: Failed to {} vApp: {}".format(action, vapp_name))
3011
3012     def get_vminstance_console(self, vm_id, console_type="vnc"):
3013         """
3014         Get a console for the virtual machine
3015         Params:
3016             vm_id: uuid of the VM
3017             console_type, can be:
3018                 "novnc" (by default), "xvpvnc" for VNC types,
3019                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
3020         Returns dict with the console parameters:
3021                 protocol: ssh, ftp, http, https, ...
3022                 server:   usually ip address
3023                 port:     the http, ssh, ... port
3024                 suffix:   extra text, e.g. the http path and query string
3025         """
3026         raise vimconn.vimconnNotImplemented("Should have implemented this")
3027
3028     # NOT USED METHODS in current version
3029
3030     def host_vim2gui(self, host, server_dict):
3031         """Transform host dictionary from VIM format to GUI format,
3032         and append to the server_dict
3033         """
3034         raise vimconn.vimconnNotImplemented("Should have implemented this")
3035
3036     def get_hosts_info(self):
3037         """Get the information of deployed hosts
3038         Returns the hosts content"""
3039         raise vimconn.vimconnNotImplemented("Should have implemented this")
3040
3041     def get_hosts(self, vim_tenant):
3042         """Get the hosts and deployed instances
3043         Returns the hosts content"""
3044         raise vimconn.vimconnNotImplemented("Should have implemented this")
3045
3046     def get_processor_rankings(self):
3047         """Get the processor rankings in the VIM database"""
3048         raise vimconn.vimconnNotImplemented("Should have implemented this")
3049
3050     def new_host(self, host_data):
3051         """Adds a new host to VIM"""
3052         '''Returns status code of the VIM response'''
3053         raise vimconn.vimconnNotImplemented("Should have implemented this")
3054
3055     def new_external_port(self, port_data):
3056         """Adds a external port to VIM"""
3057         '''Returns the port identifier'''
3058         raise vimconn.vimconnNotImplemented("Should have implemented this")
3059
3060     def new_external_network(self, net_name, net_type):
3061         """Adds a external network to VIM (shared)"""
3062         '''Returns the network identifier'''
3063         raise vimconn.vimconnNotImplemented("Should have implemented this")
3064
3065     def connect_port_network(self, port_id, network_id, admin=False):
3066         """Connects a external port to a network"""
3067         '''Returns status code of the VIM response'''
3068         raise vimconn.vimconnNotImplemented("Should have implemented this")
3069
3070     def new_vminstancefromJSON(self, vm_data):
3071         """Adds a VM instance to VIM"""
3072         '''Returns the instance identifier'''
3073         raise vimconn.vimconnNotImplemented("Should have implemented this")
3074
3075     def get_network_name_by_id(self, network_uuid=None):
3076         """Method gets vcloud director network named based on supplied uuid.
3077
3078         Args:
3079             network_uuid: network_id
3080
3081         Returns:
3082             The return network name.
3083         """
3084
3085         if not network_uuid:
3086             return None
3087
3088         try:
3089             org_dict = self.get_org(self.org_uuid)
3090             if 'networks' in org_dict:
3091                 org_network_dict = org_dict['networks']
3092                 for net_uuid in org_network_dict:
3093                     if net_uuid == network_uuid:
3094                         return org_network_dict[net_uuid]
3095         except:
3096             self.logger.debug("Exception in get_network_name_by_id")
3097             self.logger.debug(traceback.format_exc())
3098
3099         return None
3100
3101     def get_network_id_by_name(self, network_name=None):
3102         """Method gets vcloud director network uuid based on supplied name.
3103
3104         Args:
3105             network_name: network_name
3106         Returns:
3107             The return network uuid.
3108             network_uuid: network_id
3109         """
3110
3111         if not network_name:
3112             self.logger.debug("get_network_id_by_name() : Network name is empty")
3113             return None
3114
3115         try:
3116             org_dict = self.get_org(self.org_uuid)
3117             if org_dict and 'networks' in org_dict:
3118                 org_network_dict = org_dict['networks']
3119                 for net_uuid,net_name in org_network_dict.iteritems():
3120                 #For python3
3121                 #for net_uuid,net_name in org_network_dict.items():
3122                     if net_name == network_name:
3123                         return net_uuid
3124
3125         except KeyError as exp:
3126             self.logger.debug("get_network_id_by_name() : KeyError- {} ".format(exp))
3127
3128         return None
3129
3130     def list_org_action(self):
3131         """
3132         Method leverages vCloud director and query for available organization for particular user
3133
3134         Args:
3135             vca - is active VCA connection.
3136             vdc_name - is a vdc name that will be used to query vms action
3137
3138             Returns:
3139                 The return XML respond
3140         """
3141         url_list = [self.url, '/api/org']
3142         vm_list_rest_call = ''.join(url_list)
3143
3144         if self.client._session:
3145             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
3146                        'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
3147
3148             response = self.perform_request(req_type='GET',
3149                                      url=vm_list_rest_call,
3150                                            headers=headers)
3151
3152             if response.status_code == 403:
3153                 response = self.retry_rest('GET', vm_list_rest_call)
3154
3155             if response.status_code == requests.codes.ok:
3156                 return response.content
3157
3158         return None
3159
3160     def get_org_action(self, org_uuid=None):
3161         """
3162         Method leverages vCloud director and retrieve available object for organization.
3163
3164         Args:
3165             org_uuid - vCD organization uuid
3166             self.client - is active connection.
3167
3168             Returns:
3169                 The return XML respond
3170         """
3171
3172         if org_uuid is None:
3173             return None
3174
3175         url_list = [self.url, '/api/org/', org_uuid]
3176         vm_list_rest_call = ''.join(url_list)
3177
3178         if self.client._session:
3179             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
3180                      'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
3181
3182             #response = requests.get(vm_list_rest_call, headers=headers, verify=False)
3183             response = self.perform_request(req_type='GET',
3184                                             url=vm_list_rest_call,
3185                                             headers=headers)
3186             if response.status_code == 403:
3187                 response = self.retry_rest('GET', vm_list_rest_call)
3188
3189             if response.status_code == requests.codes.ok:
3190                 return response.content
3191         return None
3192
3193     def get_org(self, org_uuid=None):
3194         """
3195         Method retrieves available organization in vCloud Director
3196
3197         Args:
3198             org_uuid - is a organization uuid.
3199
3200             Returns:
3201                 The return dictionary with following key
3202                     "network" - for network list under the org
3203                     "catalogs" - for network list under the org
3204                     "vdcs" - for vdc list under org
3205         """
3206
3207         org_dict = {}
3208
3209         if org_uuid is None:
3210             return org_dict
3211
3212         content = self.get_org_action(org_uuid=org_uuid)
3213         try:
3214             vdc_list = {}
3215             network_list = {}
3216             catalog_list = {}
3217             vm_list_xmlroot = XmlElementTree.fromstring(content)
3218             for child in vm_list_xmlroot:
3219                 if child.attrib['type'] == 'application/vnd.vmware.vcloud.vdc+xml':
3220                     vdc_list[child.attrib['href'].split("/")[-1:][0]] = child.attrib['name']
3221                     org_dict['vdcs'] = vdc_list
3222                 if child.attrib['type'] == 'application/vnd.vmware.vcloud.orgNetwork+xml':
3223                     network_list[child.attrib['href'].split("/")[-1:][0]] = child.attrib['name']
3224                     org_dict['networks'] = network_list
3225                 if child.attrib['type'] == 'application/vnd.vmware.vcloud.catalog+xml':
3226                     catalog_list[child.attrib['href'].split("/")[-1:][0]] = child.attrib['name']
3227                     org_dict['catalogs'] = catalog_list
3228         except:
3229             pass
3230
3231         return org_dict
3232
3233     def get_org_list(self):
3234         """
3235         Method retrieves available organization in vCloud Director
3236
3237         Args:
3238             vca - is active VCA connection.
3239
3240             Returns:
3241                 The return dictionary and key for each entry VDC UUID
3242         """
3243
3244         org_dict = {}
3245
3246         content = self.list_org_action()
3247         try:
3248             vm_list_xmlroot = XmlElementTree.fromstring(content)
3249             for vm_xml in vm_list_xmlroot:
3250                 if vm_xml.tag.split("}")[1] == 'Org':
3251                     org_uuid = vm_xml.attrib['href'].split('/')[-1:]
3252                     org_dict[org_uuid[0]] = vm_xml.attrib['name']
3253         except:
3254             pass
3255
3256         return org_dict
3257
3258     def vms_view_action(self, vdc_name=None):
3259         """ Method leverages vCloud director vms query call
3260
3261         Args:
3262             vca - is active VCA connection.
3263             vdc_name - is a vdc name that will be used to query vms action
3264
3265             Returns:
3266                 The return XML respond
3267         """
3268         vca = self.connect()
3269         if vdc_name is None:
3270             return None
3271
3272         url_list = [vca.host, '/api/vms/query']
3273         vm_list_rest_call = ''.join(url_list)
3274
3275         if not (not vca.vcloud_session or not vca.vcloud_session.organization):
3276             refs = filter(lambda ref: ref.name == vdc_name and ref.type_ == 'application/vnd.vmware.vcloud.vdc+xml',
3277                           vca.vcloud_session.organization.Link)
3278             #For python3
3279             #refs = [ref for ref in vca.vcloud_session.organization.Link if ref.name == vdc_name and\
3280             #        ref.type_ == 'application/vnd.vmware.vcloud.vdc+xml']
3281             if len(refs) == 1:
3282                 response = Http.get(url=vm_list_rest_call,
3283                                     headers=vca.vcloud_session.get_vcloud_headers(),
3284                                     verify=vca.verify,
3285                                     logger=vca.logger)
3286                 if response.status_code == requests.codes.ok:
3287                     return response.content
3288
3289         return None
3290
3291     def get_vapp_list(self, vdc_name=None):
3292         """
3293         Method retrieves vApp list deployed vCloud director and returns a dictionary
3294         contains a list of all vapp deployed for queried VDC.
3295         The key for a dictionary is vApp UUID
3296
3297
3298         Args:
3299             vca - is active VCA connection.
3300             vdc_name - is a vdc name that will be used to query vms action
3301
3302             Returns:
3303                 The return dictionary and key for each entry vapp UUID
3304         """
3305
3306         vapp_dict = {}
3307         if vdc_name is None:
3308             return vapp_dict
3309
3310         content = self.vms_view_action(vdc_name=vdc_name)
3311         try:
3312             vm_list_xmlroot = XmlElementTree.fromstring(content)
3313             for vm_xml in vm_list_xmlroot:
3314                 if vm_xml.tag.split("}")[1] == 'VMRecord':
3315                     if vm_xml.attrib['isVAppTemplate'] == 'true':
3316                         rawuuid = vm_xml.attrib['container'].split('/')[-1:]
3317                         if 'vappTemplate-' in rawuuid[0]:
3318                             # vm in format vappTemplate-e63d40e7-4ff5-4c6d-851f-96c1e4da86a5 we remove
3319                             # vm and use raw UUID as key
3320                             vapp_dict[rawuuid[0][13:]] = vm_xml.attrib
3321         except:
3322             pass
3323
3324         return vapp_dict
3325
3326     def get_vm_list(self, vdc_name=None):
3327         """
3328         Method retrieves VM's list deployed vCloud director. It returns a dictionary
3329         contains a list of all VM's deployed for queried VDC.
3330         The key for a dictionary is VM UUID
3331
3332
3333         Args:
3334             vca - is active VCA connection.
3335             vdc_name - is a vdc name that will be used to query vms action
3336
3337             Returns:
3338                 The return dictionary and key for each entry vapp UUID
3339         """
3340         vm_dict = {}
3341
3342         if vdc_name is None:
3343             return vm_dict
3344
3345         content = self.vms_view_action(vdc_name=vdc_name)
3346         try:
3347             vm_list_xmlroot = XmlElementTree.fromstring(content)
3348             for vm_xml in vm_list_xmlroot:
3349                 if vm_xml.tag.split("}")[1] == 'VMRecord':
3350                     if vm_xml.attrib['isVAppTemplate'] == 'false':
3351                         rawuuid = vm_xml.attrib['href'].split('/')[-1:]
3352                         if 'vm-' in rawuuid[0]:
3353                             # vm in format vm-e63d40e7-4ff5-4c6d-851f-96c1e4da86a5 we remove
3354                             #  vm and use raw UUID as key
3355                             vm_dict[rawuuid[0][3:]] = vm_xml.attrib
3356         except:
3357             pass
3358
3359         return vm_dict
3360
3361     def get_vapp(self, vdc_name=None, vapp_name=None, isuuid=False):
3362         """
3363         Method retrieves VM deployed vCloud director. It returns VM attribute as dictionary
3364         contains a list of all VM's deployed for queried VDC.
3365         The key for a dictionary is VM UUID
3366
3367
3368         Args:
3369             vca - is active VCA connection.
3370             vdc_name - is a vdc name that will be used to query vms action
3371
3372             Returns:
3373                 The return dictionary and key for each entry vapp UUID
3374         """
3375         vm_dict = {}
3376         vca = self.connect()
3377         if not vca:
3378             raise vimconn.vimconnConnectionException("self.connect() is failed")
3379
3380         if vdc_name is None:
3381             return vm_dict
3382
3383         content = self.vms_view_action(vdc_name=vdc_name)
3384         try:
3385             vm_list_xmlroot = XmlElementTree.fromstring(content)
3386             for vm_xml in vm_list_xmlroot:
3387                 if vm_xml.tag.split("}")[1] == 'VMRecord' and vm_xml.attrib['isVAppTemplate'] == 'false':
3388                     # lookup done by UUID
3389                     if isuuid:
3390                         if vapp_name in vm_xml.attrib['container']:
3391                             rawuuid = vm_xml.attrib['href'].split('/')[-1:]
3392                             if 'vm-' in rawuuid[0]:
3393                                 vm_dict[rawuuid[0][3:]] = vm_xml.attrib
3394                                 break
3395                     # lookup done by Name
3396                     else:
3397                         if vapp_name in vm_xml.attrib['name']:
3398                             rawuuid = vm_xml.attrib['href'].split('/')[-1:]
3399                             if 'vm-' in rawuuid[0]:
3400                                 vm_dict[rawuuid[0][3:]] = vm_xml.attrib
3401                                 break
3402         except:
3403             pass
3404
3405         return vm_dict
3406
3407     def get_network_action(self, network_uuid=None):
3408         """
3409         Method leverages vCloud director and query network based on network uuid
3410
3411         Args:
3412             vca - is active VCA connection.
3413             network_uuid - is a network uuid
3414
3415             Returns:
3416                 The return XML respond
3417         """
3418
3419         if network_uuid is None:
3420             return None
3421
3422         url_list = [self.url, '/api/network/', network_uuid]
3423         vm_list_rest_call = ''.join(url_list)
3424
3425         if self.client._session:
3426             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
3427                      'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
3428
3429             response = self.perform_request(req_type='GET',
3430                                             url=vm_list_rest_call,
3431                                             headers=headers)
3432             #Retry login if session expired & retry sending request
3433             if response.status_code == 403:
3434                 response = self.retry_rest('GET', vm_list_rest_call)
3435
3436             if response.status_code == requests.codes.ok:
3437                 return response.content
3438
3439         return None
3440
3441     def get_vcd_network(self, network_uuid=None):
3442         """
3443         Method retrieves available network from vCloud Director
3444
3445         Args:
3446             network_uuid - is VCD network UUID
3447
3448         Each element serialized as key : value pair
3449
3450         Following keys available for access.    network_configuration['Gateway'}
3451         <Configuration>
3452           <IpScopes>
3453             <IpScope>
3454                 <IsInherited>true</IsInherited>
3455                 <Gateway>172.16.252.100</Gateway>
3456                 <Netmask>255.255.255.0</Netmask>
3457                 <Dns1>172.16.254.201</Dns1>
3458                 <Dns2>172.16.254.202</Dns2>
3459                 <DnsSuffix>vmwarelab.edu</DnsSuffix>
3460                 <IsEnabled>true</IsEnabled>
3461                 <IpRanges>
3462                     <IpRange>
3463                         <StartAddress>172.16.252.1</StartAddress>
3464                         <EndAddress>172.16.252.99</EndAddress>
3465                     </IpRange>
3466                 </IpRanges>
3467             </IpScope>
3468         </IpScopes>
3469         <FenceMode>bridged</FenceMode>
3470
3471         Returns:
3472                 The return dictionary and key for each entry vapp UUID
3473         """
3474
3475         network_configuration = {}
3476         if network_uuid is None:
3477             return network_uuid
3478
3479         try:
3480             content = self.get_network_action(network_uuid=network_uuid)
3481             vm_list_xmlroot = XmlElementTree.fromstring(content)
3482
3483             network_configuration['status'] = vm_list_xmlroot.get("status")
3484             network_configuration['name'] = vm_list_xmlroot.get("name")
3485             network_configuration['uuid'] = vm_list_xmlroot.get("id").split(":")[3]
3486
3487             for child in vm_list_xmlroot:
3488                 if child.tag.split("}")[1] == 'IsShared':
3489                     network_configuration['isShared'] = child.text.strip()
3490                 if child.tag.split("}")[1] == 'Configuration':
3491                     for configuration in child.iter():
3492                         tagKey = configuration.tag.split("}")[1].strip()
3493                         if tagKey != "":
3494                             network_configuration[tagKey] = configuration.text.strip()
3495             return network_configuration
3496         except Exception as exp :
3497             self.logger.debug("get_vcd_network: Failed with Exception {}".format(exp))
3498             raise vimconn.vimconnException("get_vcd_network: Failed with Exception {}".format(exp))
3499
3500         return network_configuration
3501
3502     def delete_network_action(self, network_uuid=None):
3503         """
3504         Method delete given network from vCloud director
3505
3506         Args:
3507             network_uuid - is a network uuid that client wish to delete
3508
3509             Returns:
3510                 The return None or XML respond or false
3511         """
3512         client = self.connect_as_admin()
3513         if not client:
3514             raise vimconn.vimconnConnectionException("Failed to connect vCD as admin")
3515         if network_uuid is None:
3516             return False
3517
3518         url_list = [self.url, '/api/admin/network/', network_uuid]
3519         vm_list_rest_call = ''.join(url_list)
3520
3521         if client._session:
3522             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
3523                      'x-vcloud-authorization': client._session.headers['x-vcloud-authorization']}
3524             response = self.perform_request(req_type='DELETE',
3525                                             url=vm_list_rest_call,
3526                                             headers=headers)
3527             if response.status_code == 202:
3528                 return True
3529
3530         return False
3531
3532     def create_network(self, network_name=None, net_type='bridge', parent_network_uuid=None,
3533                        ip_profile=None, isshared='true'):
3534         """
3535         Method create network in vCloud director
3536
3537         Args:
3538             network_name - is network name to be created.
3539             net_type - can be 'bridge','data','ptp','mgmt'.
3540             ip_profile is a dict containing the IP parameters of the network
3541             isshared - is a boolean
3542             parent_network_uuid - is parent provider vdc network that will be used for mapping.
3543             It optional attribute. by default if no parent network indicate the first available will be used.
3544
3545             Returns:
3546                 The return network uuid or return None
3547         """
3548
3549         new_network_name = [network_name, '-', str(uuid.uuid4())]
3550         content = self.create_network_rest(network_name=''.join(new_network_name),
3551                                            ip_profile=ip_profile,
3552                                            net_type=net_type,
3553                                            parent_network_uuid=parent_network_uuid,
3554                                            isshared=isshared)
3555         if content is None:
3556             self.logger.debug("Failed create network {}.".format(network_name))
3557             return None
3558
3559         try:
3560             vm_list_xmlroot = XmlElementTree.fromstring(content)
3561             vcd_uuid = vm_list_xmlroot.get('id').split(":")
3562             if len(vcd_uuid) == 4:
3563                 self.logger.info("Created new network name: {} uuid: {}".format(network_name, vcd_uuid[3]))
3564                 return vcd_uuid[3]
3565         except:
3566             self.logger.debug("Failed create network {}".format(network_name))
3567             return None
3568
3569     def create_network_rest(self, network_name=None, net_type='bridge', parent_network_uuid=None,
3570                             ip_profile=None, isshared='true'):
3571         """
3572         Method create network in vCloud director
3573
3574         Args:
3575             network_name - is network name to be created.
3576             net_type - can be 'bridge','data','ptp','mgmt'.
3577             ip_profile is a dict containing the IP parameters of the network
3578             isshared - is a boolean
3579             parent_network_uuid - is parent provider vdc network that will be used for mapping.
3580             It optional attribute. by default if no parent network indicate the first available will be used.
3581
3582             Returns:
3583                 The return network uuid or return None
3584         """
3585         client_as_admin = self.connect_as_admin()
3586         if not client_as_admin:
3587             raise vimconn.vimconnConnectionException("Failed to connect vCD.")
3588         if network_name is None:
3589             return None
3590
3591         url_list = [self.url, '/api/admin/vdc/', self.tenant_id]
3592         vm_list_rest_call = ''.join(url_list)
3593
3594         if client_as_admin._session:
3595             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
3596                      'x-vcloud-authorization': client_as_admin._session.headers['x-vcloud-authorization']}
3597
3598             response = self.perform_request(req_type='GET',
3599                                             url=vm_list_rest_call,
3600                                             headers=headers)
3601
3602             provider_network = None
3603             available_networks = None
3604             add_vdc_rest_url = None
3605
3606             if response.status_code != requests.codes.ok:
3607                 self.logger.debug("REST API call {} failed. Return status code {}".format(vm_list_rest_call,
3608                                                                                           response.status_code))
3609                 return None
3610             else:
3611                 try:
3612                     vm_list_xmlroot = XmlElementTree.fromstring(response.content)
3613                     for child in vm_list_xmlroot:
3614                         if child.tag.split("}")[1] == 'ProviderVdcReference':
3615                             provider_network = child.attrib.get('href')
3616                             # application/vnd.vmware.admin.providervdc+xml
3617                         if child.tag.split("}")[1] == 'Link':
3618                             if child.attrib.get('type') == 'application/vnd.vmware.vcloud.orgVdcNetwork+xml' \
3619                                     and child.attrib.get('rel') == 'add':
3620                                 add_vdc_rest_url = child.attrib.get('href')
3621                 except:
3622                     self.logger.debug("Failed parse respond for rest api call {}".format(vm_list_rest_call))
3623                     self.logger.debug("Respond body {}".format(response.content))
3624                     return None
3625
3626             # find  pvdc provided available network
3627             response = self.perform_request(req_type='GET',
3628                                             url=provider_network,
3629                                             headers=headers)
3630             if response.status_code != requests.codes.ok:
3631                 self.logger.debug("REST API call {} failed. Return status code {}".format(vm_list_rest_call,
3632                                                                                           response.status_code))
3633                 return None
3634
3635             if parent_network_uuid is None:
3636                 try:
3637                     vm_list_xmlroot = XmlElementTree.fromstring(response.content)
3638                     for child in vm_list_xmlroot.iter():
3639                         if child.tag.split("}")[1] == 'AvailableNetworks':
3640                             for networks in child.iter():
3641                                 # application/vnd.vmware.admin.network+xml
3642                                 if networks.attrib.get('href') is not None:
3643                                     available_networks = networks.attrib.get('href')
3644                                     break
3645                 except:
3646                     return None
3647
3648             try:
3649                 #Configure IP profile of the network
3650                 ip_profile = ip_profile if ip_profile is not None else DEFAULT_IP_PROFILE
3651
3652                 if 'subnet_address' not in ip_profile or ip_profile['subnet_address'] is None:
3653                     subnet_rand = random.randint(0, 255)
3654                     ip_base = "192.168.{}.".format(subnet_rand)
3655                     ip_profile['subnet_address'] = ip_base + "0/24"
3656                 else:
3657                     ip_base = ip_profile['subnet_address'].rsplit('.',1)[0] + '.'
3658
3659                 if 'gateway_address' not in ip_profile or ip_profile['gateway_address'] is None:
3660                     ip_profile['gateway_address']=ip_base + "1"
3661                 if 'dhcp_count' not in ip_profile or ip_profile['dhcp_count'] is None:
3662                     ip_profile['dhcp_count']=DEFAULT_IP_PROFILE['dhcp_count']
3663                 if 'dhcp_enabled' not in ip_profile or ip_profile['dhcp_enabled'] is None:
3664                     ip_profile['dhcp_enabled']=DEFAULT_IP_PROFILE['dhcp_enabled']
3665                 if 'dhcp_start_address' not in ip_profile or ip_profile['dhcp_start_address'] is None:
3666                     ip_profile['dhcp_start_address']=ip_base + "3"
3667                 if 'ip_version' not in ip_profile or ip_profile['ip_version'] is None:
3668                     ip_profile['ip_version']=DEFAULT_IP_PROFILE['ip_version']
3669                 if 'dns_address' not in ip_profile or ip_profile['dns_address'] is None:
3670                     ip_profile['dns_address']=ip_base + "2"
3671
3672                 gateway_address=ip_profile['gateway_address']
3673                 dhcp_count=int(ip_profile['dhcp_count'])
3674                 subnet_address=self.convert_cidr_to_netmask(ip_profile['subnet_address'])
3675
3676                 if ip_profile['dhcp_enabled']==True:
3677                     dhcp_enabled='true'
3678                 else:
3679                     dhcp_enabled='false'
3680                 dhcp_start_address=ip_profile['dhcp_start_address']
3681
3682                 #derive dhcp_end_address from dhcp_start_address & dhcp_count
3683                 end_ip_int = int(netaddr.IPAddress(dhcp_start_address))
3684                 end_ip_int += dhcp_count - 1
3685                 dhcp_end_address = str(netaddr.IPAddress(end_ip_int))
3686
3687                 ip_version=ip_profile['ip_version']
3688                 dns_address=ip_profile['dns_address']
3689             except KeyError as exp:
3690                 self.logger.debug("Create Network REST: Key error {}".format(exp))
3691                 raise vimconn.vimconnException("Create Network REST: Key error{}".format(exp))
3692
3693             # either use client provided UUID or search for a first available
3694             #  if both are not defined we return none
3695             if parent_network_uuid is not None:
3696                 url_list = [self.url, '/api/admin/network/', parent_network_uuid]
3697                 add_vdc_rest_url = ''.join(url_list)
3698
3699             #Creating all networks as Direct Org VDC type networks.
3700             #Unused in case of Underlay (data/ptp) network interface.
3701             fence_mode="bridged"
3702             is_inherited='false'
3703             dns_list = dns_address.split(";")
3704             dns1 = dns_list[0]
3705             dns2_text = ""
3706             if len(dns_list) >= 2:
3707                 dns2_text = "\n                                                <Dns2>{}</Dns2>\n".format(dns_list[1])
3708             data = """ <OrgVdcNetwork name="{0:s}" xmlns="http://www.vmware.com/vcloud/v1.5">
3709                             <Description>Openmano created</Description>
3710                                     <Configuration>
3711                                         <IpScopes>
3712                                             <IpScope>
3713                                                 <IsInherited>{1:s}</IsInherited>
3714                                                 <Gateway>{2:s}</Gateway>
3715                                                 <Netmask>{3:s}</Netmask>
3716                                                 <Dns1>{4:s}</Dns1>{5:s}
3717                                                 <IsEnabled>{6:s}</IsEnabled>
3718                                                 <IpRanges>
3719                                                     <IpRange>
3720                                                         <StartAddress>{7:s}</StartAddress>
3721                                                         <EndAddress>{8:s}</EndAddress>
3722                                                     </IpRange>
3723                                                 </IpRanges>
3724                                             </IpScope>
3725                                         </IpScopes>
3726                                         <ParentNetwork href="{9:s}"/>
3727                                         <FenceMode>{10:s}</FenceMode>
3728                                     </Configuration>
3729                                     <IsShared>{11:s}</IsShared>
3730                         </OrgVdcNetwork> """.format(escape(network_name), is_inherited, gateway_address,
3731                                                     subnet_address, dns1, dns2_text, dhcp_enabled,
3732                                                     dhcp_start_address, dhcp_end_address, available_networks,
3733                                                     fence_mode, isshared)
3734
3735             headers['Content-Type'] = 'application/vnd.vmware.vcloud.orgVdcNetwork+xml'
3736             try:
3737                 response = self.perform_request(req_type='POST',
3738                                            url=add_vdc_rest_url,
3739                                            headers=headers,
3740                                            data=data)
3741
3742                 if response.status_code != 201:
3743                     self.logger.debug("Create Network POST REST API call failed. Return status code {}, Response content: {}"
3744                                       .format(response.status_code,response.content))
3745                 else:
3746                     network_task = self.get_task_from_response(response.content)
3747                     self.logger.debug("Create Network REST : Waiting for Network creation complete")
3748                     time.sleep(5)
3749                     result = self.client.get_task_monitor().wait_for_success(task=network_task)
3750                     if result.get('status') == 'success':
3751                         return response.content
3752                     else:
3753                         self.logger.debug("create_network_rest task failed. Network Create response : {}"
3754                                           .format(response.content))
3755             except Exception as exp:
3756                 self.logger.debug("create_network_rest : Exception : {} ".format(exp))
3757
3758         return None
3759
3760     def convert_cidr_to_netmask(self, cidr_ip=None):
3761         """
3762         Method sets convert CIDR netmask address to normal IP format
3763         Args:
3764             cidr_ip : CIDR IP address
3765             Returns:
3766                 netmask : Converted netmask
3767         """
3768         if cidr_ip is not None:
3769             if '/' in cidr_ip:
3770                 network, net_bits = cidr_ip.split('/')
3771                 netmask = socket.inet_ntoa(struct.pack(">I", (0xffffffff << (32 - int(net_bits))) & 0xffffffff))
3772             else:
3773                 netmask = cidr_ip
3774             return netmask
3775         return None
3776
3777     def get_provider_rest(self, vca=None):
3778         """
3779         Method gets provider vdc view from vcloud director
3780
3781         Args:
3782             network_name - is network name to be created.
3783             parent_network_uuid - is parent provider vdc network that will be used for mapping.
3784             It optional attribute. by default if no parent network indicate the first available will be used.
3785
3786             Returns:
3787                 The return xml content of respond or None
3788         """
3789
3790         url_list = [self.url, '/api/admin']
3791         if vca:
3792             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
3793                        'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
3794             response = self.perform_request(req_type='GET',
3795                                             url=''.join(url_list),
3796                                             headers=headers)
3797
3798         if response.status_code == requests.codes.ok:
3799             return response.content
3800         return None
3801
3802     def create_vdc(self, vdc_name=None):
3803
3804         vdc_dict = {}
3805
3806         xml_content = self.create_vdc_from_tmpl_rest(vdc_name=vdc_name)
3807         if xml_content is not None:
3808             try:
3809                 task_resp_xmlroot = XmlElementTree.fromstring(xml_content)
3810                 for child in task_resp_xmlroot:
3811                     if child.tag.split("}")[1] == 'Owner':
3812                         vdc_id = child.attrib.get('href').split("/")[-1]
3813                         vdc_dict[vdc_id] = task_resp_xmlroot.get('href')
3814                         return vdc_dict
3815             except:
3816                 self.logger.debug("Respond body {}".format(xml_content))
3817
3818         return None
3819
3820     def create_vdc_from_tmpl_rest(self, vdc_name=None):
3821         """
3822         Method create vdc in vCloud director based on VDC template.
3823         it uses pre-defined template.
3824
3825         Args:
3826             vdc_name -  name of a new vdc.
3827
3828             Returns:
3829                 The return xml content of respond or None
3830         """
3831         # pre-requesite atleast one vdc template should be available in vCD
3832         self.logger.info("Creating new vdc {}".format(vdc_name))
3833         vca = self.connect_as_admin()
3834         if not vca:
3835             raise vimconn.vimconnConnectionException("Failed to connect vCD")
3836         if vdc_name is None:
3837             return None
3838
3839         url_list = [self.url, '/api/vdcTemplates']
3840         vm_list_rest_call = ''.join(url_list)
3841
3842         headers = {'Accept':'application/*+xml;version=' + API_VERSION,
3843                     'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
3844         response = self.perform_request(req_type='GET',
3845                                         url=vm_list_rest_call,
3846                                         headers=headers)
3847
3848         # container url to a template
3849         vdc_template_ref = None
3850         try:
3851             vm_list_xmlroot = XmlElementTree.fromstring(response.content)
3852             for child in vm_list_xmlroot:
3853                 # application/vnd.vmware.admin.providervdc+xml
3854                 # we need find a template from witch we instantiate VDC
3855                 if child.tag.split("}")[1] == 'VdcTemplate':
3856                     if child.attrib.get('type') == 'application/vnd.vmware.admin.vdcTemplate+xml':
3857                         vdc_template_ref = child.attrib.get('href')
3858         except:
3859             self.logger.debug("Failed parse respond for rest api call {}".format(vm_list_rest_call))
3860             self.logger.debug("Respond body {}".format(response.content))
3861             return None
3862
3863         # if we didn't found required pre defined template we return None
3864         if vdc_template_ref is None:
3865             return None
3866
3867         try:
3868             # instantiate vdc
3869             url_list = [self.url, '/api/org/', self.org_uuid, '/action/instantiate']
3870             vm_list_rest_call = ''.join(url_list)
3871             data = """<InstantiateVdcTemplateParams name="{0:s}" xmlns="http://www.vmware.com/vcloud/v1.5">
3872                                         <Source href="{1:s}"></Source>
3873                                         <Description>opnemano</Description>
3874                                         </InstantiateVdcTemplateParams>""".format(vdc_name, vdc_template_ref)
3875
3876             headers['Content-Type'] = 'application/vnd.vmware.vcloud.instantiateVdcTemplateParams+xml'
3877
3878             response = self.perform_request(req_type='POST',
3879                                             url=vm_list_rest_call,
3880                                             headers=headers,
3881                                             data=data)
3882
3883             vdc_task = self.get_task_from_response(response.content)
3884             self.client.get_task_monitor().wait_for_success(task=vdc_task)
3885
3886             # if we all ok we respond with content otherwise by default None
3887             if response.status_code >= 200 and response.status_code < 300:
3888                 return response.content
3889             return None
3890         except:
3891             self.logger.debug("Failed parse respond for rest api call {}".format(vm_list_rest_call))
3892             self.logger.debug("Respond body {}".format(response.content))
3893
3894         return None
3895
3896     def create_vdc_rest(self, vdc_name=None):
3897         """
3898         Method create network in vCloud director
3899
3900         Args:
3901             vdc_name - vdc name to be created
3902             Returns:
3903                 The return response
3904         """
3905
3906         self.logger.info("Creating new vdc {}".format(vdc_name))
3907
3908         vca = self.connect_as_admin()
3909         if not vca:
3910             raise vimconn.vimconnConnectionException("Failed to connect vCD")
3911         if vdc_name is None:
3912             return None
3913
3914         url_list = [self.url, '/api/admin/org/', self.org_uuid]
3915         vm_list_rest_call = ''.join(url_list)
3916
3917         if vca._session:
3918             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
3919                       'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
3920             response = self.perform_request(req_type='GET',
3921                                             url=vm_list_rest_call,
3922                                             headers=headers)
3923
3924             provider_vdc_ref = None
3925             add_vdc_rest_url = None
3926             available_networks = None
3927
3928             if response.status_code != requests.codes.ok:
3929                 self.logger.debug("REST API call {} failed. Return status code {}".format(vm_list_rest_call,
3930                                                                                           response.status_code))
3931                 return None
3932             else:
3933                 try:
3934                     vm_list_xmlroot = XmlElementTree.fromstring(response.content)
3935                     for child in vm_list_xmlroot:
3936                         # application/vnd.vmware.admin.providervdc+xml
3937                         if child.tag.split("}")[1] == 'Link':
3938                             if child.attrib.get('type') == 'application/vnd.vmware.admin.createVdcParams+xml' \
3939                                     and child.attrib.get('rel') == 'add':
3940                                 add_vdc_rest_url = child.attrib.get('href')
3941                 except:
3942                     self.logger.debug("Failed parse respond for rest api call {}".format(vm_list_rest_call))
3943                     self.logger.debug("Respond body {}".format(response.content))
3944                     return None
3945
3946                 response = self.get_provider_rest(vca=vca)
3947                 try:
3948                     vm_list_xmlroot = XmlElementTree.fromstring(response)
3949                     for child in vm_list_xmlroot:
3950                         if child.tag.split("}")[1] == 'ProviderVdcReferences':
3951                             for sub_child in child:
3952                                 provider_vdc_ref = sub_child.attrib.get('href')
3953                 except:
3954                     self.logger.debug("Failed parse respond for rest api call {}".format(vm_list_rest_call))
3955                     self.logger.debug("Respond body {}".format(response))
3956                     return None
3957
3958                 if add_vdc_rest_url is not None and provider_vdc_ref is not None:
3959                     data = """ <CreateVdcParams name="{0:s}" xmlns="http://www.vmware.com/vcloud/v1.5"><Description>{1:s}</Description>
3960                             <AllocationModel>ReservationPool</AllocationModel>
3961                             <ComputeCapacity><Cpu><Units>MHz</Units><Allocated>2048</Allocated><Limit>2048</Limit></Cpu>
3962                             <Memory><Units>MB</Units><Allocated>2048</Allocated><Limit>2048</Limit></Memory>
3963                             </ComputeCapacity><NicQuota>0</NicQuota><NetworkQuota>100</NetworkQuota>
3964                             <VdcStorageProfile><Enabled>true</Enabled><Units>MB</Units><Limit>20480</Limit><Default>true</Default></VdcStorageProfile>
3965                             <ProviderVdcReference
3966                             name="Main Provider"
3967                             href="{2:s}" />
3968                     <UsesFastProvisioning>true</UsesFastProvisioning></CreateVdcParams>""".format(escape(vdc_name),
3969                                                                                                   escape(vdc_name),
3970                                                                                                   provider_vdc_ref)
3971
3972                     headers['Content-Type'] = 'application/vnd.vmware.admin.createVdcParams+xml'
3973
3974                     response = self.perform_request(req_type='POST',
3975                                                     url=add_vdc_rest_url,
3976                                                     headers=headers,
3977                                                     data=data)
3978
3979                     # if we all ok we respond with content otherwise by default None
3980                     if response.status_code == 201:
3981                         return response.content
3982         return None
3983
3984     def get_vapp_details_rest(self, vapp_uuid=None, need_admin_access=False):
3985         """
3986         Method retrieve vapp detail from vCloud director
3987
3988         Args:
3989             vapp_uuid - is vapp identifier.
3990
3991             Returns:
3992                 The return network uuid or return None
3993         """
3994
3995         parsed_respond = {}
3996         vca = None
3997
3998         if need_admin_access:
3999             vca = self.connect_as_admin()
4000         else:
4001             vca = self.client
4002
4003         if not vca:
4004             raise vimconn.vimconnConnectionException("Failed to connect vCD")
4005         if vapp_uuid is None:
4006             return None
4007
4008         url_list = [self.url, '/api/vApp/vapp-', vapp_uuid]
4009         get_vapp_restcall = ''.join(url_list)
4010
4011         if vca._session:
4012             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
4013                        'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
4014             response = self.perform_request(req_type='GET',
4015                                             url=get_vapp_restcall,
4016                                             headers=headers)
4017
4018             if response.status_code == 403:
4019                 if need_admin_access == False:
4020                     response = self.retry_rest('GET', get_vapp_restcall)
4021
4022             if response.status_code != requests.codes.ok:
4023                 self.logger.debug("REST API call {} failed. Return status code {}".format(get_vapp_restcall,
4024                                                                                           response.status_code))
4025                 return parsed_respond
4026
4027             try:
4028                 xmlroot_respond = XmlElementTree.fromstring(response.content)
4029                 parsed_respond['ovfDescriptorUploaded'] = xmlroot_respond.attrib['ovfDescriptorUploaded']
4030
4031                 namespaces = {"vssd":"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData" ,
4032                               'ovf': 'http://schemas.dmtf.org/ovf/envelope/1',
4033                               'vmw': 'http://www.vmware.com/schema/ovf',
4034                               'vm': 'http://www.vmware.com/vcloud/v1.5',
4035                               'rasd':"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData",
4036                               "vmext":"http://www.vmware.com/vcloud/extension/v1.5",
4037                               "xmlns":"http://www.vmware.com/vcloud/v1.5"
4038                              }
4039
4040                 created_section = xmlroot_respond.find('vm:DateCreated', namespaces)
4041                 if created_section is not None:
4042                     parsed_respond['created'] = created_section.text
4043
4044                 network_section = xmlroot_respond.find('vm:NetworkConfigSection/vm:NetworkConfig', namespaces)
4045                 if network_section is not None and 'networkName' in network_section.attrib:
4046                     parsed_respond['networkname'] = network_section.attrib['networkName']
4047
4048                 ipscopes_section = \
4049                     xmlroot_respond.find('vm:NetworkConfigSection/vm:NetworkConfig/vm:Configuration/vm:IpScopes',
4050                                          namespaces)
4051                 if ipscopes_section is not None:
4052                     for ipscope in ipscopes_section:
4053                         for scope in ipscope:
4054                             tag_key = scope.tag.split("}")[1]
4055                             if tag_key == 'IpRanges':
4056                                 ip_ranges = scope.getchildren()
4057                                 for ipblock in ip_ranges:
4058                                     for block in ipblock:
4059                                         parsed_respond[block.tag.split("}")[1]] = block.text
4060                             else:
4061                                 parsed_respond[tag_key] = scope.text
4062
4063                 # parse children section for other attrib
4064                 children_section = xmlroot_respond.find('vm:Children/', namespaces)
4065                 if children_section is not None:
4066                     parsed_respond['name'] = children_section.attrib['name']
4067                     parsed_respond['nestedHypervisorEnabled'] = children_section.attrib['nestedHypervisorEnabled'] \
4068                      if  "nestedHypervisorEnabled" in children_section.attrib else None
4069                     parsed_respond['deployed'] = children_section.attrib['deployed']
4070                     parsed_respond['status'] = children_section.attrib['status']
4071                     parsed_respond['vmuuid'] = children_section.attrib['id'].split(":")[-1]
4072                     network_adapter = children_section.find('vm:NetworkConnectionSection', namespaces)
4073                     nic_list = []
4074                     for adapters in network_adapter:
4075                         adapter_key = adapters.tag.split("}")[1]
4076                         if adapter_key == 'PrimaryNetworkConnectionIndex':
4077                             parsed_respond['primarynetwork'] = adapters.text
4078                         if adapter_key == 'NetworkConnection':
4079                             vnic = {}
4080                             if 'network' in adapters.attrib:
4081                                 vnic['network'] = adapters.attrib['network']
4082                             for adapter in adapters:
4083                                 setting_key = adapter.tag.split("}")[1]
4084                                 vnic[setting_key] = adapter.text
4085                             nic_list.append(vnic)
4086
4087                     for link in children_section:
4088                         if link.tag.split("}")[1] == 'Link' and 'rel' in link.attrib:
4089                             if link.attrib['rel'] == 'screen:acquireTicket':
4090                                 parsed_respond['acquireTicket'] = link.attrib
4091                             if link.attrib['rel'] == 'screen:acquireMksTicket':
4092                                 parsed_respond['acquireMksTicket'] = link.attrib
4093
4094                     parsed_respond['interfaces'] = nic_list
4095                     vCloud_extension_section = children_section.find('xmlns:VCloudExtension', namespaces)
4096                     if vCloud_extension_section is not None:
4097                         vm_vcenter_info = {}
4098                         vim_info = vCloud_extension_section.find('vmext:VmVimInfo', namespaces)
4099                         vmext = vim_info.find('vmext:VmVimObjectRef', namespaces)
4100                         if vmext is not None:
4101                             vm_vcenter_info["vm_moref_id"] = vmext.find('vmext:MoRef', namespaces).text
4102                         parsed_respond["vm_vcenter_info"]= vm_vcenter_info
4103
4104                     virtual_hardware_section = children_section.find('ovf:VirtualHardwareSection', namespaces)
4105                     vm_virtual_hardware_info = {}
4106                     if virtual_hardware_section is not None:
4107                         for item in virtual_hardware_section.iterfind('ovf:Item',namespaces):
4108                             if item.find("rasd:Description",namespaces).text == "Hard disk":
4109                                 disk_size = item.find("rasd:HostResource" ,namespaces
4110                                                 ).attrib["{"+namespaces['vm']+"}capacity"]
4111
4112                                 vm_virtual_hardware_info["disk_size"]= disk_size
4113                                 break
4114
4115                         for link in virtual_hardware_section:
4116                             if link.tag.split("}")[1] == 'Link' and 'rel' in link.attrib:
4117                                 if link.attrib['rel'] == 'edit' and link.attrib['href'].endswith("/disks"):
4118                                     vm_virtual_hardware_info["disk_edit_href"] = link.attrib['href']
4119                                     break
4120
4121                     parsed_respond["vm_virtual_hardware"]= vm_virtual_hardware_info
4122             except Exception as exp :
4123                 self.logger.info("Error occurred calling rest api for getting vApp details {}".format(exp))
4124         return parsed_respond
4125
4126     def acquire_console(self, vm_uuid=None):
4127
4128         if vm_uuid is None:
4129             return None
4130         if self.client._session:
4131             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
4132                        'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
4133             vm_dict = self.get_vapp_details_rest(vapp_uuid=vm_uuid)
4134             console_dict = vm_dict['acquireTicket']
4135             console_rest_call = console_dict['href']
4136
4137             response = self.perform_request(req_type='POST',
4138                                             url=console_rest_call,
4139                                             headers=headers)
4140
4141             if response.status_code == 403:
4142                 response = self.retry_rest('POST', console_rest_call)
4143
4144             if response.status_code == requests.codes.ok:
4145                 return response.content
4146
4147         return None
4148
4149     def modify_vm_disk(self, vapp_uuid, flavor_disk):
4150         """
4151         Method retrieve vm disk details
4152
4153         Args:
4154             vapp_uuid - is vapp identifier.
4155             flavor_disk - disk size as specified in VNFD (flavor)
4156
4157             Returns:
4158                 The return network uuid or return None
4159         """
4160         status = None
4161         try:
4162             #Flavor disk is in GB convert it into MB
4163             flavor_disk = int(flavor_disk) * 1024
4164             vm_details = self.get_vapp_details_rest(vapp_uuid)
4165             if vm_details:
4166                 vm_name = vm_details["name"]
4167                 self.logger.info("VM: {} flavor_disk :{}".format(vm_name , flavor_disk))
4168
4169             if vm_details and "vm_virtual_hardware" in vm_details:
4170                 vm_disk = int(vm_details["vm_virtual_hardware"]["disk_size"])
4171                 disk_edit_href = vm_details["vm_virtual_hardware"]["disk_edit_href"]
4172
4173                 self.logger.info("VM: {} VM_disk :{}".format(vm_name , vm_disk))
4174
4175                 if flavor_disk > vm_disk:
4176                     status = self.modify_vm_disk_rest(disk_edit_href ,flavor_disk)
4177                     self.logger.info("Modify disk of VM {} from {} to {} MB".format(vm_name,
4178                                                          vm_disk,  flavor_disk ))
4179                 else:
4180                     status = True
4181                     self.logger.info("No need to modify disk of VM {}".format(vm_name))
4182
4183             return status
4184         except Exception as exp:
4185             self.logger.info("Error occurred while modifing disk size {}".format(exp))
4186
4187
4188     def modify_vm_disk_rest(self, disk_href , disk_size):
4189         """
4190         Method retrieve modify vm disk size
4191
4192         Args:
4193             disk_href - vCD API URL to GET and PUT disk data
4194             disk_size - disk size as specified in VNFD (flavor)
4195
4196             Returns:
4197                 The return network uuid or return None
4198         """
4199         if disk_href is None or disk_size is None:
4200             return None
4201
4202         if self.client._session:
4203                 headers = {'Accept':'application/*+xml;version=' + API_VERSION,
4204                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
4205                 response = self.perform_request(req_type='GET',
4206                                                 url=disk_href,
4207                                                 headers=headers)
4208
4209         if response.status_code == 403:
4210             response = self.retry_rest('GET', disk_href)
4211
4212         if response.status_code != requests.codes.ok:
4213             self.logger.debug("GET REST API call {} failed. Return status code {}".format(disk_href,
4214                                                                             response.status_code))
4215             return None
4216         try:
4217             lxmlroot_respond = lxmlElementTree.fromstring(response.content)
4218             namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.iteritems() if prefix}
4219             #For python3
4220             #namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.items() if prefix}
4221             namespaces["xmlns"]= "http://www.vmware.com/vcloud/v1.5"
4222
4223             for item in lxmlroot_respond.iterfind('xmlns:Item',namespaces):
4224                 if item.find("rasd:Description",namespaces).text == "Hard disk":
4225                     disk_item = item.find("rasd:HostResource" ,namespaces )
4226                     if disk_item is not None:
4227                         disk_item.attrib["{"+namespaces['xmlns']+"}capacity"] = str(disk_size)
4228                         break
4229
4230             data = lxmlElementTree.tostring(lxmlroot_respond, encoding='utf8', method='xml',
4231                                              xml_declaration=True)
4232
4233             #Send PUT request to modify disk size
4234             headers['Content-Type'] = 'application/vnd.vmware.vcloud.rasdItemsList+xml; charset=ISO-8859-1'
4235
4236             response = self.perform_request(req_type='PUT',
4237                                                 url=disk_href,
4238                                                 headers=headers,
4239                                                 data=data)
4240             if response.status_code == 403:
4241                 add_headers = {'Content-Type': headers['Content-Type']}
4242                 response = self.retry_rest('PUT', disk_href, add_headers, data)
4243
4244             if response.status_code != 202:
4245                 self.logger.debug("PUT REST API call {} failed. Return status code {}".format(disk_href,
4246                                                                             response.status_code))
4247             else:
4248                 modify_disk_task = self.get_task_from_response(response.content)
4249                 result = self.client.get_task_monitor().wait_for_success(task=modify_disk_task)
4250                 if result.get('status') == 'success':
4251                     return True
4252                 else:
4253                     return False
4254             return None
4255
4256         except Exception as exp :
4257                 self.logger.info("Error occurred calling rest api for modifing disk size {}".format(exp))
4258                 return None
4259
4260     def add_pci_devices(self, vapp_uuid , pci_devices , vmname_andid):
4261         """
4262             Method to attach pci devices to VM
4263
4264              Args:
4265                 vapp_uuid - uuid of vApp/VM
4266                 pci_devices - pci devices infromation as specified in VNFD (flavor)
4267
4268             Returns:
4269                 The status of add pci device task , vm object and
4270                 vcenter_conect object
4271         """
4272         vm_obj = None
4273         self.logger.info("Add pci devices {} into vApp {}".format(pci_devices , vapp_uuid))
4274         vcenter_conect, content = self.get_vcenter_content()
4275         vm_moref_id = self.get_vm_moref_id(vapp_uuid)
4276
4277         if vm_moref_id:
4278             try:
4279                 no_of_pci_devices = len(pci_devices)
4280                 if no_of_pci_devices > 0:
4281                     #Get VM and its host
4282                     host_obj, vm_obj = self.get_vm_obj(content, vm_moref_id)
4283                     self.logger.info("VM {} is currently on host {}".format(vm_obj, host_obj))
4284                     if host_obj and vm_obj:
4285                         #get PCI devies from host on which vapp is currently installed
4286                         avilable_pci_devices = self.get_pci_devices(host_obj, no_of_pci_devices)
4287
4288                         if avilable_pci_devices is None:
4289                             #find other hosts with active pci devices
4290                             new_host_obj , avilable_pci_devices = self.get_host_and_PCIdevices(
4291                                                                 content,
4292                                                                 no_of_pci_devices
4293                                                                 )
4294
4295                             if new_host_obj is not None and avilable_pci_devices is not None and len(avilable_pci_devices)> 0:
4296                                 #Migrate vm to the host where PCI devices are availble
4297                                 self.logger.info("Relocate VM {} on new host {}".format(vm_obj, new_host_obj))
4298                                 task = self.relocate_vm(new_host_obj, vm_obj)
4299                                 if task is not None:
4300                                     result = self.wait_for_vcenter_task(task, vcenter_conect)
4301                                     self.logger.info("Migrate VM status: {}".format(result))
4302                                     host_obj = new_host_obj
4303                                 else:
4304                                     self.logger.info("Fail to migrate VM : {}".format(result))
4305                                     raise vimconn.vimconnNotFoundException(
4306                                     "Fail to migrate VM : {} to host {}".format(
4307                                                     vmname_andid,
4308                                                     new_host_obj)
4309                                         )
4310
4311                         if host_obj is not None and avilable_pci_devices is not None and len(avilable_pci_devices)> 0:
4312                             #Add PCI devices one by one
4313                             for pci_device in avilable_pci_devices:
4314                                 task = self.add_pci_to_vm(host_obj, vm_obj, pci_device)
4315                                 if task:
4316                                     status= self.wait_for_vcenter_task(task, vcenter_conect)
4317                                     if status:
4318                                         self.logger.info("Added PCI device {} to VM {}".format(pci_device,str(vm_obj)))
4319                                 else:
4320                                     self.logger.error("Fail to add PCI device {} to VM {}".format(pci_device,str(vm_obj)))
4321                             return True, vm_obj, vcenter_conect
4322                         else:
4323                             self.logger.error("Currently there is no host with"\
4324                                               " {} number of avaialble PCI devices required for VM {}".format(
4325                                                                             no_of_pci_devices,
4326                                                                             vmname_andid)
4327                                               )
4328                             raise vimconn.vimconnNotFoundException(
4329                                     "Currently there is no host with {} "\
4330                                     "number of avaialble PCI devices required for VM {}".format(
4331                                                                             no_of_pci_devices,
4332                                                                             vmname_andid))
4333                 else:
4334                     self.logger.debug("No infromation about PCI devices {} ",pci_devices)
4335
4336             except vmodl.MethodFault as error:
4337                 self.logger.error("Error occurred while adding PCI devices {} ",error)
4338         return None, vm_obj, vcenter_conect
4339
4340     def get_vm_obj(self, content, mob_id):
4341         """
4342             Method to get the vsphere VM object associated with a given morf ID
4343              Args:
4344                 vapp_uuid - uuid of vApp/VM
4345                 content - vCenter content object
4346                 mob_id - mob_id of VM
4347
4348             Returns:
4349                     VM and host object
4350         """
4351         vm_obj = None
4352         host_obj = None
4353         try :
4354             container = content.viewManager.CreateContainerView(content.rootFolder,
4355                                                         [vim.VirtualMachine], True
4356                                                         )
4357             for vm in container.view:
4358                 mobID = vm._GetMoId()
4359                 if mobID == mob_id:
4360                     vm_obj = vm
4361                     host_obj = vm_obj.runtime.host
4362                     break
4363         except Exception as exp:
4364             self.logger.error("Error occurred while finding VM object : {}".format(exp))
4365         return host_obj, vm_obj
4366
4367     def get_pci_devices(self, host, need_devices):
4368         """
4369             Method to get the details of pci devices on given host
4370              Args:
4371                 host - vSphere host object
4372                 need_devices - number of pci devices needed on host
4373
4374              Returns:
4375                 array of pci devices
4376         """
4377         all_devices = []
4378         all_device_ids = []
4379         used_devices_ids = []
4380
4381         try:
4382             if host:
4383                 pciPassthruInfo = host.config.pciPassthruInfo
4384                 pciDevies = host.hardware.pciDevice
4385
4386             for pci_status in pciPassthruInfo:
4387                 if pci_status.passthruActive:
4388                     for device in pciDevies:
4389                         if device.id == pci_status.id:
4390                             all_device_ids.append(device.id)
4391                             all_devices.append(device)
4392
4393             #check if devices are in use
4394             avalible_devices = all_devices
4395             for vm in host.vm:
4396                 if vm.runtime.powerState == vim.VirtualMachinePowerState.poweredOn:
4397                     vm_devices = vm.config.hardware.device
4398                     for device in vm_devices:
4399                         if type(device) is vim.vm.device.VirtualPCIPassthrough:
4400                             if device.backing.id in all_device_ids:
4401                                 for use_device in avalible_devices:
4402                                     if use_device.id == device.backing.id:
4403                                         avalible_devices.remove(use_device)
4404                                 used_devices_ids.append(device.backing.id)
4405                                 self.logger.debug("Device {} from devices {}"\
4406                                         "is in use".format(device.backing.id,
4407                                                            device)
4408                                             )
4409             if len(avalible_devices) < need_devices:
4410                 self.logger.debug("Host {} don't have {} number of active devices".format(host,
4411                                                                             need_devices))
4412                 self.logger.debug("found only {} devives {}".format(len(avalible_devices),
4413                                                                     avalible_devices))
4414                 return None
4415             else:
4416                 required_devices = avalible_devices[:need_devices]
4417                 self.logger.info("Found {} PCI devivces on host {} but required only {}".format(
4418                                                             len(avalible_devices),
4419                                                             host,
4420                                                             need_devices))
4421                 self.logger.info("Retruning {} devices as {}".format(need_devices,
4422                                                                 required_devices ))
4423                 return required_devices
4424
4425         except Exception as exp:
4426             self.logger.error("Error {} occurred while finding pci devices on host: {}".format(exp, host))
4427
4428         return None
4429
4430     def get_host_and_PCIdevices(self, content, need_devices):
4431         """
4432          Method to get the details of pci devices infromation on all hosts
4433
4434             Args:
4435                 content - vSphere host object
4436                 need_devices - number of pci devices needed on host
4437
4438             Returns:
4439                  array of pci devices and host object
4440         """
4441         host_obj = None
4442         pci_device_objs = None
4443         try:
4444             if content:
4445                 container = content.viewManager.CreateContainerView(content.rootFolder,
4446                                                             [vim.HostSystem], True)
4447                 for host in container.view:
4448                     devices = self.get_pci_devices(host, need_devices)
4449                     if devices:
4450                         host_obj = host
4451                         pci_device_objs = devices
4452                         break
4453         except Exception as exp:
4454             self.logger.error("Error {} occurred while finding pci devices on host: {}".format(exp, host_obj))
4455
4456         return host_obj,pci_device_objs
4457
4458     def relocate_vm(self, dest_host, vm) :
4459         """
4460          Method to get the relocate VM to new host
4461
4462             Args:
4463                 dest_host - vSphere host object
4464                 vm - vSphere VM object
4465
4466             Returns:
4467                 task object
4468         """
4469         task = None
4470         try:
4471             relocate_spec = vim.vm.RelocateSpec(host=dest_host)
4472             task = vm.Relocate(relocate_spec)
4473             self.logger.info("Migrating {} to destination host {}".format(vm, dest_host))
4474         except Exception as exp:
4475             self.logger.error("Error occurred while relocate VM {} to new host {}: {}".format(
4476                                                                             dest_host, vm, exp))
4477         return task
4478
4479     def wait_for_vcenter_task(self, task, actionName='job', hideResult=False):
4480         """
4481         Waits and provides updates on a vSphere task
4482         """
4483         while task.info.state == vim.TaskInfo.State.running:
4484             time.sleep(2)
4485
4486         if task.info.state == vim.TaskInfo.State.success:
4487             if task.info.result is not None and not hideResult:
4488                 self.logger.info('{} completed successfully, result: {}'.format(
4489                                                             actionName,
4490                                                             task.info.result))
4491             else:
4492                 self.logger.info('Task {} completed successfully.'.format(actionName))
4493         else:
4494             self.logger.error('{} did not complete successfully: {} '.format(
4495                                                             actionName,
4496                                                             task.info.error)
4497                               )
4498
4499         return task.info.result
4500
4501     def add_pci_to_vm(self,host_object, vm_object, host_pci_dev):
4502         """
4503          Method to add pci device in given VM
4504
4505             Args:
4506                 host_object - vSphere host object
4507                 vm_object - vSphere VM object
4508                 host_pci_dev -  host_pci_dev must be one of the devices from the
4509                                 host_object.hardware.pciDevice list
4510                                 which is configured as a PCI passthrough device
4511
4512             Returns:
4513                 task object
4514         """
4515         task = None
4516         if vm_object and host_object and host_pci_dev:
4517             try :
4518                 #Add PCI device to VM
4519                 pci_passthroughs = vm_object.environmentBrowser.QueryConfigTarget(host=None).pciPassthrough
4520                 systemid_by_pciid = {item.pciDevice.id: item.systemId for item in pci_passthroughs}
4521
4522                 if host_pci_dev.id not in systemid_by_pciid:
4523                     self.logger.error("Device {} is not a passthrough device ".format(host_pci_dev))
4524                     return None
4525
4526                 deviceId = hex(host_pci_dev.deviceId % 2**16).lstrip('0x')
4527                 backing = vim.VirtualPCIPassthroughDeviceBackingInfo(deviceId=deviceId,
4528                                             id=host_pci_dev.id,
4529                                             systemId=systemid_by_pciid[host_pci_dev.id],
4530                                             vendorId=host_pci_dev.vendorId,
4531                                             deviceName=host_pci_dev.deviceName)
4532
4533                 hba_object = vim.VirtualPCIPassthrough(key=-100, backing=backing)
4534
4535                 new_device_config = vim.VirtualDeviceConfigSpec(device=hba_object)
4536                 new_device_config.operation = "add"
4537                 vmConfigSpec = vim.vm.ConfigSpec()
4538                 vmConfigSpec.deviceChange = [new_device_config]
4539
4540                 task = vm_object.ReconfigVM_Task(spec=vmConfigSpec)
4541                 self.logger.info("Adding PCI device {} into VM {} from host {} ".format(
4542                                                             host_pci_dev, vm_object, host_object)
4543                                 )
4544             except Exception as exp:
4545                 self.logger.error("Error occurred while adding pci devive {} to VM {}: {}".format(
4546                                                                             host_pci_dev,
4547                                                                             vm_object,
4548                                                                              exp))
4549         return task
4550
4551     def get_vm_vcenter_info(self):
4552         """
4553         Method to get details of vCenter and vm
4554
4555             Args:
4556                 vapp_uuid - uuid of vApp or VM
4557
4558             Returns:
4559                 Moref Id of VM and deails of vCenter
4560         """
4561         vm_vcenter_info = {}
4562
4563         if self.vcenter_ip is not None:
4564             vm_vcenter_info["vm_vcenter_ip"] = self.vcenter_ip
4565         else:
4566             raise vimconn.vimconnException(message="vCenter IP is not provided."\
4567                                            " Please provide vCenter IP while attaching datacenter to tenant in --config")
4568         if self.vcenter_port is not None:
4569             vm_vcenter_info["vm_vcenter_port"] = self.vcenter_port
4570         else:
4571             raise vimconn.vimconnException(message="vCenter port is not provided."\
4572                                            " Please provide vCenter port while attaching datacenter to tenant in --config")
4573         if self.vcenter_user is not None:
4574             vm_vcenter_info["vm_vcenter_user"] = self.vcenter_user
4575         else:
4576             raise vimconn.vimconnException(message="vCenter user is not provided."\
4577                                            " Please provide vCenter user while attaching datacenter to tenant in --config")
4578
4579         if self.vcenter_password is not None:
4580             vm_vcenter_info["vm_vcenter_password"] = self.vcenter_password
4581         else:
4582             raise vimconn.vimconnException(message="vCenter user password is not provided."\
4583                                            " Please provide vCenter user password while attaching datacenter to tenant in --config")
4584
4585         return vm_vcenter_info
4586
4587
4588     def get_vm_pci_details(self, vmuuid):
4589         """
4590             Method to get VM PCI device details from vCenter
4591
4592             Args:
4593                 vm_obj - vSphere VM object
4594
4595             Returns:
4596                 dict of PCI devives attached to VM
4597
4598         """
4599         vm_pci_devices_info = {}
4600         try:
4601             vcenter_conect, content = self.get_vcenter_content()
4602             vm_moref_id = self.get_vm_moref_id(vmuuid)
4603             if vm_moref_id:
4604                 #Get VM and its host
4605                 if content:
4606                     host_obj, vm_obj = self.get_vm_obj(content, vm_moref_id)
4607                     if host_obj and vm_obj:
4608                         vm_pci_devices_info["host_name"]= host_obj.name
4609                         vm_pci_devices_info["host_ip"]= host_obj.config.network.vnic[0].spec.ip.ipAddress
4610                         for device in vm_obj.config.hardware.device:
4611                             if type(device) == vim.vm.device.VirtualPCIPassthrough:
4612                                 device_details={'devide_id':device.backing.id,
4613                                                 'pciSlotNumber':device.slotInfo.pciSlotNumber,
4614                                             }
4615                                 vm_pci_devices_info[device.deviceInfo.label] = device_details
4616                 else:
4617                     self.logger.error("Can not connect to vCenter while getting "\
4618                                           "PCI devices infromationn")
4619                 return vm_pci_devices_info
4620         except Exception as exp:
4621             self.logger.error("Error occurred while getting VM infromationn"\
4622                              " for VM : {}".format(exp))
4623             raise vimconn.vimconnException(message=exp)
4624
4625     def add_network_adapter_to_vms(self, vapp, network_name, primary_nic_index, nicIndex, net, nic_type=None):
4626         """
4627             Method to add network adapter type to vm
4628             Args :
4629                 network_name - name of network
4630                 primary_nic_index - int value for primary nic index
4631                 nicIndex - int value for nic index
4632                 nic_type - specify model name to which add to vm
4633             Returns:
4634                 None
4635         """
4636
4637         try:
4638             ip_address = None
4639             floating_ip = False
4640             mac_address = None
4641             if 'floating_ip' in net: floating_ip = net['floating_ip']
4642
4643             # Stub for ip_address feature
4644             if 'ip_address' in net: ip_address = net['ip_address']
4645
4646             if 'mac_address' in net: mac_address = net['mac_address']
4647
4648             if floating_ip:
4649                 allocation_mode = "POOL"
4650             elif ip_address:
4651                 allocation_mode = "MANUAL"
4652             else:
4653                 allocation_mode = "DHCP"
4654
4655             if not nic_type:
4656                 for vms in vapp.get_all_vms():
4657                     vm_id = vms.get('id').split(':')[-1]
4658
4659                     url_rest_call = "{}/api/vApp/vm-{}/networkConnectionSection/".format(self.url, vm_id)
4660
4661                     headers = {'Accept':'application/*+xml;version=' + API_VERSION,
4662                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
4663                     response = self.perform_request(req_type='GET',
4664                                                     url=url_rest_call,
4665                                                     headers=headers)
4666
4667                     if response.status_code == 403:
4668                         response = self.retry_rest('GET', url_rest_call)
4669
4670                     if response.status_code != 200:
4671                         self.logger.error("REST call {} failed reason : {}"\
4672                                              "status code : {}".format(url_rest_call,
4673                                                                     response.content,
4674                                                                response.status_code))
4675                         raise vimconn.vimconnException("add_network_adapter_to_vms : Failed to get "\
4676                                                                          "network connection section")
4677
4678                     data = response.content
4679                     data = data.split('<Link rel="edit"')[0]
4680                     if '<PrimaryNetworkConnectionIndex>' not in data:
4681                         item = """<PrimaryNetworkConnectionIndex>{}</PrimaryNetworkConnectionIndex>
4682                                 <NetworkConnection network="{}">
4683                                 <NetworkConnectionIndex>{}</NetworkConnectionIndex>
4684                                 <IsConnected>true</IsConnected>
4685                                 <IpAddressAllocationMode>{}</IpAddressAllocationMode>
4686                                 </NetworkConnection>""".format(primary_nic_index, network_name, nicIndex,
4687                                                                                          allocation_mode)
4688                         # Stub for ip_address feature
4689                         if ip_address:
4690                             ip_tag = '<IpAddress>{}</IpAddress>'.format(ip_address)
4691                             item =  item.replace('</NetworkConnectionIndex>\n','</NetworkConnectionIndex>\n{}\n'.format(ip_tag))
4692
4693                         if mac_address:
4694                             mac_tag = '<MACAddress>{}</MACAddress>'.format(mac_address)
4695                             item =  item.replace('</IsConnected>\n','</IsConnected>\n{}\n'.format(mac_tag))
4696
4697                         data = data.replace('</ovf:Info>\n','</ovf:Info>\n{}\n</NetworkConnectionSection>'.format(item))
4698                     else:
4699                         new_item = """<NetworkConnection network="{}">
4700                                     <NetworkConnectionIndex>{}</NetworkConnectionIndex>
4701                                     <IsConnected>true</IsConnected>
4702                                     <IpAddressAllocationMode>{}</IpAddressAllocationMode>
4703                                     </NetworkConnection>""".format(network_name, nicIndex,
4704                                                                           allocation_mode)
4705                         # Stub for ip_address feature
4706                         if ip_address:
4707                             ip_tag = '<IpAddress>{}</IpAddress>'.format(ip_address)
4708                             new_item =  new_item.replace('</NetworkConnectionIndex>\n','</NetworkConnectionIndex>\n{}\n'.format(ip_tag))
4709
4710                         if mac_address:
4711                             mac_tag = '<MACAddress>{}</MACAddress>'.format(mac_address)
4712                             new_item =  new_item.replace('</IsConnected>\n','</IsConnected>\n{}\n'.format(mac_tag))
4713
4714                         data = data + new_item + '</NetworkConnectionSection>'
4715
4716                     headers['Content-Type'] = 'application/vnd.vmware.vcloud.networkConnectionSection+xml'
4717
4718                     response = self.perform_request(req_type='PUT',
4719                                                     url=url_rest_call,
4720                                                     headers=headers,
4721                                                     data=data)
4722
4723                     if response.status_code == 403:
4724                         add_headers = {'Content-Type': headers['Content-Type']}
4725                         response = self.retry_rest('PUT', url_rest_call, add_headers, data)
4726
4727                     if response.status_code != 202:
4728                         self.logger.error("REST call {} failed reason : {}"\
4729                                             "status code : {} ".format(url_rest_call,
4730                                                                     response.content,
4731                                                                response.status_code))
4732                         raise vimconn.vimconnException("add_network_adapter_to_vms : Failed to update "\
4733                                                                             "network connection section")
4734                     else:
4735                         nic_task = self.get_task_from_response(response.content)
4736                         result = self.client.get_task_monitor().wait_for_success(task=nic_task)
4737                         if result.get('status') == 'success':
4738                             self.logger.info("add_network_adapter_to_vms(): VM {} conneced to "\
4739                                                                "default NIC type".format(vm_id))
4740                         else:
4741                             self.logger.error("add_network_adapter_to_vms(): VM {} failed to "\
4742                                                               "connect NIC type".format(vm_id))
4743             else:
4744                 for vms in vapp.get_all_vms():
4745                     vm_id = vms.get('id').split(':')[-1]
4746
4747
4748                     url_rest_call = "{}/api/vApp/vm-{}/networkConnectionSection/".format(self.url, vm_id)
4749
4750                     headers = {'Accept':'application/*+xml;version=' + API_VERSION,
4751                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
4752                     response = self.perform_request(req_type='GET',
4753                                                     url=url_rest_call,
4754                                                     headers=headers)
4755
4756                     if response.status_code == 403:
4757                         response = self.retry_rest('GET', url_rest_call)
4758
4759                     if response.status_code != 200:
4760                         self.logger.error("REST call {} failed reason : {}"\
4761                                             "status code : {}".format(url_rest_call,
4762                                                                    response.content,
4763                                                               response.status_code))
4764                         raise vimconn.vimconnException("add_network_adapter_to_vms : Failed to get "\
4765                                                                         "network connection section")
4766                     data = response.content
4767                     data = data.split('<Link rel="edit"')[0]
4768                     if '<PrimaryNetworkConnectionIndex>' not in data:
4769                         item = """<PrimaryNetworkConnectionIndex>{}</PrimaryNetworkConnectionIndex>
4770                                 <NetworkConnection network="{}">
4771                                 <NetworkConnectionIndex>{}</NetworkConnectionIndex>
4772                                 <IsConnected>true</IsConnected>
4773                                 <IpAddressAllocationMode>{}</IpAddressAllocationMode>
4774                                 <NetworkAdapterType>{}</NetworkAdapterType>
4775                                 </NetworkConnection>""".format(primary_nic_index, network_name, nicIndex,
4776                                                                                allocation_mode, nic_type)
4777                         # Stub for ip_address feature
4778                         if ip_address:
4779                             ip_tag = '<IpAddress>{}</IpAddress>'.format(ip_address)
4780                             item =  item.replace('</NetworkConnectionIndex>\n','</NetworkConnectionIndex>\n{}\n'.format(ip_tag))
4781
4782                         if mac_address:
4783                             mac_tag = '<MACAddress>{}</MACAddress>'.format(mac_address)
4784                             item = item.replace('</IsConnected>\n','</IsConnected>\n{}\n'.format(mac_tag))
4785
4786                         data = data.replace('</ovf:Info>\n','</ovf:Info>\n{}\n</NetworkConnectionSection>'.format(item))
4787                     else:
4788                         new_item = """<NetworkConnection network="{}">
4789                                     <NetworkConnectionIndex>{}</NetworkConnectionIndex>
4790                                     <IsConnected>true</IsConnected>
4791                                     <IpAddressAllocationMode>{}</IpAddressAllocationMode>
4792                                     <NetworkAdapterType>{}</NetworkAdapterType>
4793                                     </NetworkConnection>""".format(network_name, nicIndex,
4794                                                                 allocation_mode, nic_type)
4795                         # Stub for ip_address feature
4796                         if ip_address:
4797                             ip_tag = '<IpAddress>{}</IpAddress>'.format(ip_address)
4798                             new_item =  new_item.replace('</NetworkConnectionIndex>\n','</NetworkConnectionIndex>\n{}\n'.format(ip_tag))
4799
4800                         if mac_address:
4801                             mac_tag = '<MACAddress>{}</MACAddress>'.format(mac_address)
4802                             new_item =  new_item.replace('</IsConnected>\n','</IsConnected>\n{}\n'.format(mac_tag))
4803
4804                         data = data + new_item + '</NetworkConnectionSection>'
4805
4806                     headers['Content-Type'] = 'application/vnd.vmware.vcloud.networkConnectionSection+xml'
4807
4808                     response = self.perform_request(req_type='PUT',
4809                                                     url=url_rest_call,
4810                                                     headers=headers,
4811                                                     data=data)
4812
4813                     if response.status_code == 403:
4814                         add_headers = {'Content-Type': headers['Content-Type']}
4815                         response = self.retry_rest('PUT', url_rest_call, add_headers, data)
4816
4817                     if response.status_code != 202:
4818                         self.logger.error("REST call {} failed reason : {}"\
4819                                             "status code : {}".format(url_rest_call,
4820                                                                    response.content,
4821                                                               response.status_code))
4822                         raise vimconn.vimconnException("add_network_adapter_to_vms : Failed to update "\
4823                                                                            "network connection section")
4824                     else:
4825                         nic_task = self.get_task_from_response(response.content)
4826                         result = self.client.get_task_monitor().wait_for_success(task=nic_task)
4827                         if result.get('status') == 'success':
4828                             self.logger.info("add_network_adapter_to_vms(): VM {} "\
4829                                                "conneced to NIC type {}".format(vm_id, nic_type))
4830                         else:
4831                             self.logger.error("add_network_adapter_to_vms(): VM {} "\
4832                                                "failed to connect NIC type {}".format(vm_id, nic_type))
4833         except Exception as exp:
4834             self.logger.error("add_network_adapter_to_vms() : exception occurred "\
4835                                                "while adding Network adapter")
4836             raise vimconn.vimconnException(message=exp)
4837
4838
4839     def set_numa_affinity(self, vmuuid, paired_threads_id):
4840         """
4841             Method to assign numa affinity in vm configuration parammeters
4842             Args :
4843                 vmuuid - vm uuid
4844                 paired_threads_id - one or more virtual processor
4845                                     numbers
4846             Returns:
4847                 return if True
4848         """
4849         try:
4850             vcenter_conect, content = self.get_vcenter_content()
4851             vm_moref_id = self.get_vm_moref_id(vmuuid)
4852
4853             host_obj, vm_obj = self.get_vm_obj(content ,vm_moref_id)
4854             if vm_obj:
4855                 config_spec = vim.vm.ConfigSpec()
4856                 config_spec.extraConfig = []
4857                 opt = vim.option.OptionValue()
4858                 opt.key = 'numa.nodeAffinity'
4859                 opt.value = str(paired_threads_id)
4860                 config_spec.extraConfig.append(opt)
4861                 task = vm_obj.ReconfigVM_Task(config_spec)
4862                 if task:
4863                     result = self.wait_for_vcenter_task(task, vcenter_conect)
4864                     extra_config = vm_obj.config.extraConfig
4865                     flag = False
4866                     for opts in extra_config:
4867                         if 'numa.nodeAffinity' in opts.key:
4868                             flag = True
4869                             self.logger.info("set_numa_affinity: Sucessfully assign numa affinity "\
4870                                                      "value {} for vm {}".format(opt.value, vm_obj))
4871                         if flag:
4872                             return
4873             else:
4874                 self.logger.error("set_numa_affinity: Failed to assign numa affinity")
4875         except Exception as exp:
4876             self.logger.error("set_numa_affinity : exception occurred while setting numa affinity "\
4877                                                        "for VM {} : {}".format(vm_obj, vm_moref_id))
4878             raise vimconn.vimconnException("set_numa_affinity : Error {} failed to assign numa "\
4879                                                                            "affinity".format(exp))
4880
4881
4882     def cloud_init(self, vapp, cloud_config):
4883         """
4884         Method to inject ssh-key
4885         vapp - vapp object
4886         cloud_config a dictionary with:
4887                 'key-pairs': (optional) list of strings with the public key to be inserted to the default user
4888                 'users': (optional) list of users to be inserted, each item is a dict with:
4889                     'name': (mandatory) user name,
4890                     'key-pairs': (optional) list of strings with the public key to be inserted to the user
4891                 'user-data': (optional) can be a string with the text script to be passed directly to cloud-init,
4892                     or a list of strings, each one contains a script to be passed, usually with a MIMEmultipart file
4893                 'config-files': (optional). List of files to be transferred. Each item is a dict with:
4894                     'dest': (mandatory) string with the destination absolute path
4895                     'encoding': (optional, by default text). Can be one of:
4896                         'b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64'
4897                     'content' (mandatory): string with the content of the file
4898                     'permissions': (optional) string with file permissions, typically octal notation '0644'
4899                     'owner': (optional) file owner, string with the format 'owner:group'
4900                 'boot-data-drive': boolean to indicate if user-data must be passed using a boot drive (hard disk
4901         """
4902         try:
4903             if not isinstance(cloud_config, dict):
4904                 raise Exception("cloud_init : parameter cloud_config is not a dictionary")
4905             else:
4906                 key_pairs = []
4907                 userdata = []
4908                 if "key-pairs" in cloud_config:
4909                     key_pairs = cloud_config["key-pairs"]
4910
4911                 if "users" in cloud_config:
4912                     userdata = cloud_config["users"]
4913
4914                 self.logger.debug("cloud_init : Guest os customization started..")
4915                 customize_script = self.format_script(key_pairs=key_pairs, users_list=userdata)
4916                 customize_script = customize_script.replace("&","&amp;")
4917                 self.guest_customization(vapp, customize_script)
4918
4919         except Exception as exp:
4920             self.logger.error("cloud_init : exception occurred while injecting "\
4921                                                                        "ssh-key")
4922             raise vimconn.vimconnException("cloud_init : Error {} failed to inject "\
4923                                                                "ssh-key".format(exp))
4924
4925     def format_script(self, key_pairs=[], users_list=[]):
4926         bash_script = """#!/bin/sh
4927         echo performing customization tasks with param $1 at `date "+DATE: %Y-%m-%d - TIME: %H:%M:%S"` >> /root/customization.log
4928         if [ "$1" = "precustomization" ];then
4929             echo performing precustomization tasks   on `date "+DATE: %Y-%m-%d - TIME: %H:%M:%S"` >> /root/customization.log
4930         """
4931
4932         keys = "\n".join(key_pairs)
4933         if keys:
4934             keys_data = """
4935             if [ ! -d /root/.ssh ];then
4936                 mkdir /root/.ssh
4937                 chown root:root /root/.ssh
4938                 chmod 700 /root/.ssh
4939                 touch /root/.ssh/authorized_keys
4940                 chown root:root /root/.ssh/authorized_keys
4941                 chmod 600 /root/.ssh/authorized_keys
4942                 # make centos with selinux happy
4943                 which restorecon && restorecon -Rv /root/.ssh
4944             else
4945                 touch /root/.ssh/authorized_keys
4946                 chown root:root /root/.ssh/authorized_keys
4947                 chmod 600 /root/.ssh/authorized_keys
4948             fi
4949             echo '{key}' >> /root/.ssh/authorized_keys
4950             """.format(key=keys)
4951
4952             bash_script+= keys_data
4953
4954         for user in users_list:
4955             if 'name' in user: user_name = user['name']
4956             if 'key-pairs' in user:
4957                 user_keys = "\n".join(user['key-pairs'])
4958             else:
4959                 user_keys = None
4960
4961             add_user_name = """
4962                 useradd -d /home/{user_name} -m -g users -s /bin/bash {user_name}
4963                 """.format(user_name=user_name)
4964
4965             bash_script+= add_user_name
4966
4967             if user_keys:
4968                 user_keys_data = """
4969                 mkdir /home/{user_name}/.ssh
4970                 chown {user_name}:{user_name} /home/{user_name}/.ssh
4971                 chmod 700 /home/{user_name}/.ssh
4972                 touch /home/{user_name}/.ssh/authorized_keys
4973                 chown {user_name}:{user_name} /home/{user_name}/.ssh/authorized_keys
4974                 chmod 600 /home/{user_name}/.ssh/authorized_keys
4975                 # make centos with selinux happy
4976                 which restorecon && restorecon -Rv /home/{user_name}/.ssh
4977                 echo '{user_key}' >> /home/{user_name}/.ssh/authorized_keys
4978                 """.format(user_name=user_name,user_key=user_keys)
4979
4980                 bash_script+= user_keys_data
4981
4982         return bash_script+"\n\tfi"
4983
4984     def guest_customization(self, vapp, customize_script):
4985         """
4986         Method to customize guest os
4987         vapp - Vapp object
4988         customize_script - Customize script to be run at first boot of VM.
4989         """
4990         for vm in vapp.get_all_vms():
4991             vm_id = vm.get('id').split(':')[-1]
4992             vm_name = vm.get('name')
4993             vm_name = vm_name.replace('_','-')
4994
4995             vm_customization_url = "{}/api/vApp/vm-{}/guestCustomizationSection/".format(self.url, vm_id)
4996             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
4997                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
4998
4999             headers['Content-Type'] = "application/vnd.vmware.vcloud.guestCustomizationSection+xml"
5000
5001             data = """<GuestCustomizationSection
5002                            xmlns="http://www.vmware.com/vcloud/v1.5"
5003                            xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/1"
5004                            ovf:required="false" href="{}" type="application/vnd.vmware.vcloud.guestCustomizationSection+xml">
5005                            <ovf:Info>Specifies Guest OS Customization Settings</ovf:Info>
5006                            <Enabled>true</Enabled>
5007                            <ChangeSid>false</ChangeSid>
5008                            <VirtualMachineId>{}</VirtualMachineId>
5009                            <JoinDomainEnabled>false</JoinDomainEnabled>
5010                            <UseOrgSettings>false</UseOrgSettings>
5011                            <AdminPasswordEnabled>false</AdminPasswordEnabled>
5012                            <AdminPasswordAuto>true</AdminPasswordAuto>
5013                            <AdminAutoLogonEnabled>false</AdminAutoLogonEnabled>
5014                            <AdminAutoLogonCount>0</AdminAutoLogonCount>
5015                            <ResetPasswordRequired>false</ResetPasswordRequired>
5016                            <CustomizationScript>{}</CustomizationScript>
5017                            <ComputerName>{}</ComputerName>
5018                            <Link href="{}" type="application/vnd.vmware.vcloud.guestCustomizationSection+xml" rel="edit"/>
5019                        </GuestCustomizationSection>
5020                    """.format(vm_customization_url,
5021                                              vm_id,
5022                                   customize_script,
5023                                            vm_name,
5024                               vm_customization_url)
5025
5026             response = self.perform_request(req_type='PUT',
5027                                              url=vm_customization_url,
5028                                              headers=headers,
5029                                              data=data)
5030             if response.status_code == 202:
5031                 guest_task = self.get_task_from_response(response.content)
5032                 self.client.get_task_monitor().wait_for_success(task=guest_task)
5033                 self.logger.info("guest_customization : customized guest os task "\
5034                                              "completed for VM {}".format(vm_name))
5035             else:
5036                 self.logger.error("guest_customization : task for customized guest os"\
5037                                                     "failed for VM {}".format(vm_name))
5038                 raise vimconn.vimconnException("guest_customization : failed to perform"\
5039                                        "guest os customization on VM {}".format(vm_name))
5040
5041     def add_new_disk(self, vapp_uuid, disk_size):
5042         """
5043             Method to create an empty vm disk
5044
5045             Args:
5046                 vapp_uuid - is vapp identifier.
5047                 disk_size - size of disk to be created in GB
5048
5049             Returns:
5050                 None
5051         """
5052         status = False
5053         vm_details = None
5054         try:
5055             #Disk size in GB, convert it into MB
5056             if disk_size is not None:
5057                 disk_size_mb = int(disk_size) * 1024
5058                 vm_details = self.get_vapp_details_rest(vapp_uuid)
5059
5060             if vm_details and "vm_virtual_hardware" in vm_details:
5061                 self.logger.info("Adding disk to VM: {} disk size:{}GB".format(vm_details["name"], disk_size))
5062                 disk_href = vm_details["vm_virtual_hardware"]["disk_edit_href"]
5063                 status = self.add_new_disk_rest(disk_href, disk_size_mb)
5064
5065         except Exception as exp:
5066             msg = "Error occurred while creating new disk {}.".format(exp)
5067             self.rollback_newvm(vapp_uuid, msg)
5068
5069         if status:
5070             self.logger.info("Added new disk to VM: {} disk size:{}GB".format(vm_details["name"], disk_size))
5071         else:
5072             #If failed to add disk, delete VM
5073             msg = "add_new_disk: Failed to add new disk to {}".format(vm_details["name"])
5074             self.rollback_newvm(vapp_uuid, msg)
5075
5076
5077     def add_new_disk_rest(self, disk_href, disk_size_mb):
5078         """
5079         Retrives vApp Disks section & add new empty disk
5080
5081         Args:
5082             disk_href: Disk section href to addd disk
5083             disk_size_mb: Disk size in MB
5084
5085             Returns: Status of add new disk task
5086         """
5087         status = False
5088         if self.client._session:
5089             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
5090                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
5091             response = self.perform_request(req_type='GET',
5092                                             url=disk_href,
5093                                             headers=headers)
5094
5095         if response.status_code == 403:
5096             response = self.retry_rest('GET', disk_href)
5097
5098         if response.status_code != requests.codes.ok:
5099             self.logger.error("add_new_disk_rest: GET REST API call {} failed. Return status code {}"
5100                               .format(disk_href, response.status_code))
5101             return status
5102         try:
5103             #Find but type & max of instance IDs assigned to disks
5104             lxmlroot_respond = lxmlElementTree.fromstring(response.content)
5105             namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.iteritems() if prefix}
5106             #For python3
5107             #namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.items() if prefix}
5108             namespaces["xmlns"]= "http://www.vmware.com/vcloud/v1.5"
5109             instance_id = 0
5110             for item in lxmlroot_respond.iterfind('xmlns:Item',namespaces):
5111                 if item.find("rasd:Description",namespaces).text == "Hard disk":
5112                     inst_id = int(item.find("rasd:InstanceID" ,namespaces).text)
5113                     if inst_id > instance_id:
5114                         instance_id = inst_id
5115                         disk_item = item.find("rasd:HostResource" ,namespaces)
5116                         bus_subtype = disk_item.attrib["{"+namespaces['xmlns']+"}busSubType"]
5117                         bus_type = disk_item.attrib["{"+namespaces['xmlns']+"}busType"]
5118
5119             instance_id = instance_id + 1
5120             new_item =   """<Item>
5121                                 <rasd:Description>Hard disk</rasd:Description>
5122                                 <rasd:ElementName>New disk</rasd:ElementName>
5123                                 <rasd:HostResource
5124                                     xmlns:vcloud="http://www.vmware.com/vcloud/v1.5"
5125                                     vcloud:capacity="{}"
5126                                     vcloud:busSubType="{}"
5127                                     vcloud:busType="{}"></rasd:HostResource>
5128                                 <rasd:InstanceID>{}</rasd:InstanceID>
5129                                 <rasd:ResourceType>17</rasd:ResourceType>
5130                             </Item>""".format(disk_size_mb, bus_subtype, bus_type, instance_id)
5131
5132             new_data = response.content
5133             #Add new item at the bottom
5134             new_data = new_data.replace('</Item>\n</RasdItemsList>', '</Item>\n{}\n</RasdItemsList>'.format(new_item))
5135
5136             # Send PUT request to modify virtual hardware section with new disk
5137             headers['Content-Type'] = 'application/vnd.vmware.vcloud.rasdItemsList+xml; charset=ISO-8859-1'
5138
5139             response = self.perform_request(req_type='PUT',
5140                                             url=disk_href,
5141                                             data=new_data,
5142                                             headers=headers)
5143
5144             if response.status_code == 403:
5145                 add_headers = {'Content-Type': headers['Content-Type']}
5146                 response = self.retry_rest('PUT', disk_href, add_headers, new_data)
5147
5148             if response.status_code != 202:
5149                 self.logger.error("PUT REST API call {} failed. Return status code {}. Response Content:{}"
5150                                   .format(disk_href, response.status_code, response.content))
5151             else:
5152                 add_disk_task = self.get_task_from_response(response.content)
5153                 result = self.client.get_task_monitor().wait_for_success(task=add_disk_task)
5154                 if result.get('status') == 'success':
5155                     status = True
5156                 else:
5157                     self.logger.error("Add new disk REST task failed to add {} MB disk".format(disk_size_mb))
5158
5159         except Exception as exp:
5160             self.logger.error("Error occurred calling rest api for creating new disk {}".format(exp))
5161
5162         return status
5163
5164
5165     def add_existing_disk(self, catalogs=None, image_id=None, size=None, template_name=None, vapp_uuid=None):
5166         """
5167             Method to add existing disk to vm
5168             Args :
5169                 catalogs - List of VDC catalogs
5170                 image_id - Catalog ID
5171                 template_name - Name of template in catalog
5172                 vapp_uuid - UUID of vApp
5173             Returns:
5174                 None
5175         """
5176         disk_info = None
5177         vcenter_conect, content = self.get_vcenter_content()
5178         #find moref-id of vm in image
5179         catalog_vm_info = self.get_vapp_template_details(catalogs=catalogs,
5180                                                          image_id=image_id,
5181                                                         )
5182
5183         if catalog_vm_info and "vm_vcenter_info" in catalog_vm_info:
5184             if "vm_moref_id" in catalog_vm_info["vm_vcenter_info"]:
5185                 catalog_vm_moref_id = catalog_vm_info["vm_vcenter_info"].get("vm_moref_id", None)
5186                 if catalog_vm_moref_id:
5187                     self.logger.info("Moref_id of VM in catalog : {}" .format(catalog_vm_moref_id))
5188                     host, catalog_vm_obj = self.get_vm_obj(content, catalog_vm_moref_id)
5189                     if catalog_vm_obj:
5190                         #find existing disk
5191                         disk_info = self.find_disk(catalog_vm_obj)
5192                     else:
5193                         exp_msg = "No VM with image id {} found".format(image_id)
5194                         self.rollback_newvm(vapp_uuid, exp_msg, exp_type="NotFound")
5195         else:
5196             exp_msg = "No Image found with image ID {} ".format(image_id)
5197             self.rollback_newvm(vapp_uuid, exp_msg, exp_type="NotFound")
5198
5199         if disk_info:
5200             self.logger.info("Existing disk_info : {}".format(disk_info))
5201             #get VM
5202             vm_moref_id = self.get_vm_moref_id(vapp_uuid)
5203             host, vm_obj = self.get_vm_obj(content, vm_moref_id)
5204             if vm_obj:
5205                 status = self.add_disk(vcenter_conect=vcenter_conect,
5206                                        vm=vm_obj,
5207                                        disk_info=disk_info,
5208                                        size=size,
5209                                        vapp_uuid=vapp_uuid
5210                                        )
5211             if status:
5212                 self.logger.info("Disk from image id {} added to {}".format(image_id,
5213                                                                             vm_obj.config.name)
5214                                  )
5215         else:
5216             msg = "No disk found with image id {} to add in VM {}".format(
5217                                                             image_id,
5218                                                             vm_obj.config.name)
5219             self.rollback_newvm(vapp_uuid, msg, exp_type="NotFound")
5220
5221
5222     def find_disk(self, vm_obj):
5223         """
5224          Method to find details of existing disk in VM
5225             Args :
5226                 vm_obj - vCenter object of VM
5227                 image_id - Catalog ID
5228             Returns:
5229                 disk_info : dict of disk details
5230         """
5231         disk_info = {}
5232         if vm_obj:
5233             try:
5234                 devices = vm_obj.config.hardware.device
5235                 for device in devices:
5236                     if type(device) is vim.vm.device.VirtualDisk:
5237                         if isinstance(device.backing,vim.vm.device.VirtualDisk.FlatVer2BackingInfo) and hasattr(device.backing, 'fileName'):
5238                             disk_info["full_path"] = device.backing.fileName
5239                             disk_info["datastore"] = device.backing.datastore
5240                             disk_info["capacityKB"] = device.capacityInKB
5241                             break
5242             except Exception as exp:
5243                 self.logger.error("find_disk() : exception occurred while "\
5244                                   "getting existing disk details :{}".format(exp))
5245         return disk_info
5246
5247
5248     def add_disk(self, vcenter_conect=None, vm=None, size=None, vapp_uuid=None, disk_info={}):
5249         """
5250          Method to add existing disk in VM
5251             Args :
5252                 vcenter_conect - vCenter content object
5253                 vm - vCenter vm object
5254                 disk_info : dict of disk details
5255             Returns:
5256                 status : status of add disk task
5257         """
5258         datastore = disk_info["datastore"] if "datastore" in disk_info else None
5259         fullpath = disk_info["full_path"] if "full_path" in disk_info else None
5260         capacityKB = disk_info["capacityKB"] if "capacityKB" in disk_info else None
5261         if size is not None:
5262             #Convert size from GB to KB
5263             sizeKB = int(size) * 1024 * 1024
5264             #compare size of existing disk and user given size.Assign whicherver is greater
5265             self.logger.info("Add Existing disk : sizeKB {} , capacityKB {}".format(
5266                                                                     sizeKB, capacityKB))
5267             if sizeKB > capacityKB:
5268                 capacityKB = sizeKB
5269
5270         if datastore and fullpath and capacityKB:
5271             try:
5272                 spec = vim.vm.ConfigSpec()
5273                 # get all disks on a VM, set unit_number to the next available
5274                 unit_number = 0
5275                 for dev in vm.config.hardware.device:
5276                     if hasattr(dev.backing, 'fileName'):
5277                         unit_number = int(dev.unitNumber) + 1
5278                         # unit_number 7 reserved for scsi controller
5279                         if unit_number == 7:
5280                             unit_number += 1
5281                     if isinstance(dev, vim.vm.device.VirtualDisk):
5282                         #vim.vm.device.VirtualSCSIController
5283                         controller_key = dev.controllerKey
5284
5285                 self.logger.info("Add Existing disk : unit number {} , controller key {}".format(
5286                                                                     unit_number, controller_key))
5287                 # add disk here
5288                 dev_changes = []
5289                 disk_spec = vim.vm.device.VirtualDeviceSpec()
5290                 disk_spec.operation = vim.vm.device.VirtualDeviceSpec.Operation.add
5291                 disk_spec.device = vim.vm.device.VirtualDisk()
5292                 disk_spec.device.backing = \
5293                     vim.vm.device.VirtualDisk.FlatVer2BackingInfo()
5294                 disk_spec.device.backing.thinProvisioned = True
5295                 disk_spec.device.backing.diskMode = 'persistent'
5296                 disk_spec.device.backing.datastore  = datastore
5297                 disk_spec.device.backing.fileName  = fullpath
5298
5299                 disk_spec.device.unitNumber = unit_number
5300                 disk_spec.device.capacityInKB = capacityKB
5301                 disk_spec.device.controllerKey = controller_key
5302                 dev_changes.append(disk_spec)
5303                 spec.deviceChange = dev_changes
5304                 task = vm.ReconfigVM_Task(spec=spec)
5305                 status = self.wait_for_vcenter_task(task, vcenter_conect)
5306                 return status
5307             except Exception as exp:
5308                 exp_msg = "add_disk() : exception {} occurred while adding disk "\
5309                           "{} to vm {}".format(exp,
5310                                                fullpath,
5311                                                vm.config.name)
5312                 self.rollback_newvm(vapp_uuid, exp_msg)
5313         else:
5314             msg = "add_disk() : Can not add disk to VM with disk info {} ".format(disk_info)
5315             self.rollback_newvm(vapp_uuid, msg)
5316
5317
5318     def get_vcenter_content(self):
5319         """
5320          Get the vsphere content object
5321         """
5322         try:
5323             vm_vcenter_info = self.get_vm_vcenter_info()
5324         except Exception as exp:
5325             self.logger.error("Error occurred while getting vCenter infromationn"\
5326                              " for VM : {}".format(exp))
5327             raise vimconn.vimconnException(message=exp)
5328
5329         context = None
5330         if hasattr(ssl, '_create_unverified_context'):
5331             context = ssl._create_unverified_context()
5332
5333         vcenter_conect = SmartConnect(
5334                     host=vm_vcenter_info["vm_vcenter_ip"],
5335                     user=vm_vcenter_info["vm_vcenter_user"],
5336                     pwd=vm_vcenter_info["vm_vcenter_password"],
5337                     port=int(vm_vcenter_info["vm_vcenter_port"]),
5338                     sslContext=context
5339                 )
5340         atexit.register(Disconnect, vcenter_conect)
5341         content = vcenter_conect.RetrieveContent()
5342         return vcenter_conect, content
5343
5344
5345     def get_vm_moref_id(self, vapp_uuid):
5346         """
5347         Get the moref_id of given VM
5348         """
5349         try:
5350             if vapp_uuid:
5351                 vm_details = self.get_vapp_details_rest(vapp_uuid, need_admin_access=True)
5352                 if vm_details and "vm_vcenter_info" in vm_details:
5353                     vm_moref_id = vm_details["vm_vcenter_info"].get("vm_moref_id", None)
5354             return vm_moref_id
5355
5356         except Exception as exp:
5357             self.logger.error("Error occurred while getting VM moref ID "\
5358                              " for VM : {}".format(exp))
5359             return None
5360
5361
5362     def get_vapp_template_details(self, catalogs=None, image_id=None , template_name=None):
5363         """
5364             Method to get vApp template details
5365                 Args :
5366                     catalogs - list of VDC catalogs
5367                     image_id - Catalog ID to find
5368                     template_name : template name in catalog
5369                 Returns:
5370                     parsed_respond : dict of vApp tempalte details
5371         """
5372         parsed_response = {}
5373
5374         vca = self.connect_as_admin()
5375         if not vca:
5376             raise vimconn.vimconnConnectionException("Failed to connect vCD")
5377
5378         try:
5379             org, vdc = self.get_vdc_details()
5380             catalog = self.get_catalog_obj(image_id, catalogs)
5381             if catalog:
5382                 items = org.get_catalog_item(catalog.get('name'), catalog.get('name'))
5383                 catalog_items = [items.attrib]
5384
5385                 if len(catalog_items) == 1:
5386                     headers = {'Accept':'application/*+xml;version=' + API_VERSION,
5387                            'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
5388
5389                     response = self.perform_request(req_type='GET',
5390                                                     url=catalog_items[0].get('href'),
5391                                                     headers=headers)
5392                     catalogItem = XmlElementTree.fromstring(response.content)
5393                     entity = [child for child in catalogItem if child.get("type") == "application/vnd.vmware.vcloud.vAppTemplate+xml"][0]
5394                     vapp_tempalte_href = entity.get("href")
5395                     #get vapp details and parse moref id
5396
5397                     namespaces = {"vssd":"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData" ,
5398                                   'ovf': 'http://schemas.dmtf.org/ovf/envelope/1',
5399                                   'vmw': 'http://www.vmware.com/schema/ovf',
5400                                   'vm': 'http://www.vmware.com/vcloud/v1.5',
5401                                   'rasd':"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData",
5402                                   'vmext':"http://www.vmware.com/vcloud/extension/v1.5",
5403                                   'xmlns':"http://www.vmware.com/vcloud/v1.5"
5404                                 }
5405
5406                     if vca._session:
5407                         response = self.perform_request(req_type='GET',
5408                                                     url=vapp_tempalte_href,
5409                                                     headers=headers)
5410
5411                         if response.status_code != requests.codes.ok:
5412                             self.logger.debug("REST API call {} failed. Return status code {}".format(
5413                                                 vapp_tempalte_href, response.status_code))
5414
5415                         else:
5416                             xmlroot_respond = XmlElementTree.fromstring(response.content)
5417                             children_section = xmlroot_respond.find('vm:Children/', namespaces)
5418                             if children_section is not None:
5419                                 vCloud_extension_section = children_section.find('xmlns:VCloudExtension', namespaces)
5420                             if vCloud_extension_section is not None:
5421                                 vm_vcenter_info = {}
5422                                 vim_info = vCloud_extension_section.find('vmext:VmVimInfo', namespaces)
5423                                 vmext = vim_info.find('vmext:VmVimObjectRef', namespaces)
5424                                 if vmext is not None:
5425                                     vm_vcenter_info["vm_moref_id"] = vmext.find('vmext:MoRef', namespaces).text
5426                                 parsed_response["vm_vcenter_info"]= vm_vcenter_info
5427
5428         except Exception as exp :
5429             self.logger.info("Error occurred calling rest api for getting vApp details {}".format(exp))
5430
5431         return parsed_response
5432
5433
5434     def rollback_newvm(self, vapp_uuid, msg , exp_type="Genric"):
5435         """
5436             Method to delete vApp
5437                 Args :
5438                     vapp_uuid - vApp UUID
5439                     msg - Error message to be logged
5440                     exp_type : Exception type
5441                 Returns:
5442                     None
5443         """
5444         if vapp_uuid:
5445             status = self.delete_vminstance(vapp_uuid)
5446         else:
5447             msg = "No vApp ID"
5448         self.logger.error(msg)
5449         if exp_type == "Genric":
5450             raise vimconn.vimconnException(msg)
5451         elif exp_type == "NotFound":
5452             raise vimconn.vimconnNotFoundException(message=msg)
5453
5454     def add_sriov(self, vapp_uuid, sriov_nets, vmname_andid):
5455         """
5456             Method to attach SRIOV adapters to VM
5457
5458              Args:
5459                 vapp_uuid - uuid of vApp/VM
5460                 sriov_nets - SRIOV devices infromation as specified in VNFD (flavor)
5461                 vmname_andid - vmname
5462
5463             Returns:
5464                 The status of add SRIOV adapter task , vm object and
5465                 vcenter_conect object
5466         """
5467         vm_obj = None
5468         vcenter_conect, content = self.get_vcenter_content()
5469         vm_moref_id = self.get_vm_moref_id(vapp_uuid)
5470
5471         if vm_moref_id:
5472             try:
5473                 no_of_sriov_devices = len(sriov_nets)
5474                 if no_of_sriov_devices > 0:
5475                     #Get VM and its host
5476                     host_obj, vm_obj = self.get_vm_obj(content, vm_moref_id)
5477                     self.logger.info("VM {} is currently on host {}".format(vm_obj, host_obj))
5478                     if host_obj and vm_obj:
5479                         #get SRIOV devies from host on which vapp is currently installed
5480                         avilable_sriov_devices = self.get_sriov_devices(host_obj,
5481                                                                 no_of_sriov_devices,
5482                                                                 )
5483
5484                         if len(avilable_sriov_devices) == 0:
5485                             #find other hosts with active pci devices
5486                             new_host_obj , avilable_sriov_devices = self.get_host_and_sriov_devices(
5487                                                                 content,
5488                                                                 no_of_sriov_devices,
5489                                                                 )
5490
5491                             if new_host_obj is not None and len(avilable_sriov_devices)> 0:
5492                                 #Migrate vm to the host where SRIOV devices are available
5493                                 self.logger.info("Relocate VM {} on new host {}".format(vm_obj,
5494                                                                                     new_host_obj))
5495                                 task = self.relocate_vm(new_host_obj, vm_obj)
5496                                 if task is not None:
5497                                     result = self.wait_for_vcenter_task(task, vcenter_conect)
5498                                     self.logger.info("Migrate VM status: {}".format(result))
5499                                     host_obj = new_host_obj
5500                                 else:
5501                                     self.logger.info("Fail to migrate VM : {}".format(result))
5502                                     raise vimconn.vimconnNotFoundException(
5503                                     "Fail to migrate VM : {} to host {}".format(
5504                                                     vmname_andid,
5505                                                     new_host_obj)
5506                                         )
5507
5508                         if host_obj is not None and avilable_sriov_devices is not None and len(avilable_sriov_devices)> 0:
5509                             #Add SRIOV devices one by one
5510                             for sriov_net in sriov_nets:
5511                                 network_name = sriov_net.get('net_id')
5512                                 dvs_portgr_name = self.create_dvPort_group(network_name)
5513                                 if sriov_net.get('type') == "VF" or sriov_net.get('type') == "SR-IOV":
5514                                     #add vlan ID ,Modify portgroup for vlan ID
5515                                     self.configure_vlanID(content, vcenter_conect, network_name)
5516
5517                                 task = self.add_sriov_to_vm(content,
5518                                                             vm_obj,
5519                                                             host_obj,
5520                                                             network_name,
5521                                                             avilable_sriov_devices[0]
5522                                                             )
5523                                 if task:
5524                                     status= self.wait_for_vcenter_task(task, vcenter_conect)
5525                                     if status:
5526                                         self.logger.info("Added SRIOV {} to VM {}".format(
5527                                                                         no_of_sriov_devices,
5528                                                                         str(vm_obj)))
5529                                 else:
5530                                     self.logger.error("Fail to add SRIOV {} to VM {}".format(
5531                                                                         no_of_sriov_devices,
5532                                                                         str(vm_obj)))
5533                                     raise vimconn.vimconnUnexpectedResponse(
5534                                     "Fail to add SRIOV adapter in VM ".format(str(vm_obj))
5535                                         )
5536                             return True, vm_obj, vcenter_conect
5537                         else:
5538                             self.logger.error("Currently there is no host with"\
5539                                               " {} number of avaialble SRIOV "\
5540                                               "VFs required for VM {}".format(
5541                                                                 no_of_sriov_devices,
5542                                                                 vmname_andid)
5543                                               )
5544                             raise vimconn.vimconnNotFoundException(
5545                                     "Currently there is no host with {} "\
5546                                     "number of avaialble SRIOV devices required for VM {}".format(
5547                                                                             no_of_sriov_devices,
5548                                                                             vmname_andid))
5549                 else:
5550                     self.logger.debug("No infromation about SRIOV devices {} ",sriov_nets)
5551
5552             except vmodl.MethodFault as error:
5553                 self.logger.error("Error occurred while adding SRIOV {} ",error)
5554         return None, vm_obj, vcenter_conect
5555
5556
5557     def get_sriov_devices(self,host, no_of_vfs):
5558         """
5559             Method to get the details of SRIOV devices on given host
5560              Args:
5561                 host - vSphere host object
5562                 no_of_vfs - number of VFs needed on host
5563
5564              Returns:
5565                 array of SRIOV devices
5566         """
5567         sriovInfo=[]
5568         if host:
5569             for device in host.config.pciPassthruInfo:
5570                 if isinstance(device,vim.host.SriovInfo) and device.sriovActive:
5571                     if device.numVirtualFunction >= no_of_vfs:
5572                         sriovInfo.append(device)
5573                         break
5574         return sriovInfo
5575
5576
5577     def get_host_and_sriov_devices(self, content, no_of_vfs):
5578         """
5579          Method to get the details of SRIOV devices infromation on all hosts
5580
5581             Args:
5582                 content - vSphere host object
5583                 no_of_vfs - number of pci VFs needed on host
5584
5585             Returns:
5586                  array of SRIOV devices and host object
5587         """
5588         host_obj = None
5589         sriov_device_objs = None
5590         try:
5591             if content:
5592                 container = content.viewManager.CreateContainerView(content.rootFolder,
5593                                                             [vim.HostSystem], True)
5594                 for host in container.view:
5595                     devices = self.get_sriov_devices(host, no_of_vfs)
5596                     if devices:
5597                         host_obj = host
5598                         sriov_device_objs = devices
5599                         break
5600         except Exception as exp:
5601             self.logger.error("Error {} occurred while finding SRIOV devices on host: {}".format(exp, host_obj))
5602
5603         return host_obj,sriov_device_objs
5604
5605
5606     def add_sriov_to_vm(self,content, vm_obj, host_obj, network_name, sriov_device):
5607         """
5608          Method to add SRIOV adapter to vm
5609
5610             Args:
5611                 host_obj - vSphere host object
5612                 vm_obj - vSphere vm object
5613                 content - vCenter content object
5614                 network_name - name of distributed virtaul portgroup
5615                 sriov_device - SRIOV device info
5616
5617             Returns:
5618                  task object
5619         """
5620         devices = []
5621         vnic_label = "sriov nic"
5622         try:
5623             dvs_portgr = self.get_dvport_group(network_name)
5624             network_name = dvs_portgr.name
5625             nic = vim.vm.device.VirtualDeviceSpec()
5626             # VM device
5627             nic.operation = vim.vm.device.VirtualDeviceSpec.Operation.add
5628             nic.device = vim.vm.device.VirtualSriovEthernetCard()
5629             nic.device.addressType = 'assigned'
5630             #nic.device.key = 13016
5631             nic.device.deviceInfo = vim.Description()
5632             nic.device.deviceInfo.label = vnic_label
5633             nic.device.deviceInfo.summary = network_name
5634             nic.device.backing = vim.vm.device.VirtualEthernetCard.NetworkBackingInfo()
5635
5636             nic.device.backing.network = self.get_obj(content, [vim.Network], network_name)
5637             nic.device.backing.deviceName = network_name
5638             nic.device.backing.useAutoDetect = False
5639             nic.device.connectable = vim.vm.device.VirtualDevice.ConnectInfo()
5640             nic.device.connectable.startConnected = True
5641             nic.device.connectable.allowGuestControl = True
5642
5643             nic.device.sriovBacking = vim.vm.device.VirtualSriovEthernetCard.SriovBackingInfo()
5644             nic.device.sriovBacking.physicalFunctionBacking = vim.vm.device.VirtualPCIPassthrough.DeviceBackingInfo()
5645             nic.device.sriovBacking.physicalFunctionBacking.id = sriov_device.id
5646
5647             devices.append(nic)
5648             vmconf = vim.vm.ConfigSpec(deviceChange=devices)
5649             task = vm_obj.ReconfigVM_Task(vmconf)
5650             return task
5651         except Exception as exp:
5652             self.logger.error("Error {} occurred while adding SRIOV adapter in VM: {}".format(exp, vm_obj))
5653             return None
5654
5655
5656     def create_dvPort_group(self, network_name):
5657         """
5658          Method to create disributed virtual portgroup
5659
5660             Args:
5661                 network_name - name of network/portgroup
5662
5663             Returns:
5664                 portgroup key
5665         """
5666         try:
5667             new_network_name = [network_name, '-', str(uuid.uuid4())]
5668             network_name=''.join(new_network_name)
5669             vcenter_conect, content = self.get_vcenter_content()
5670
5671             dv_switch = self.get_obj(content, [vim.DistributedVirtualSwitch], self.dvs_name)
5672             if dv_switch:
5673                 dv_pg_spec = vim.dvs.DistributedVirtualPortgroup.ConfigSpec()
5674                 dv_pg_spec.name = network_name
5675
5676                 dv_pg_spec.type = vim.dvs.DistributedVirtualPortgroup.PortgroupType.earlyBinding
5677                 dv_pg_spec.defaultPortConfig = vim.dvs.VmwareDistributedVirtualSwitch.VmwarePortConfigPolicy()
5678                 dv_pg_spec.defaultPortConfig.securityPolicy = vim.dvs.VmwareDistributedVirtualSwitch.SecurityPolicy()
5679                 dv_pg_spec.defaultPortConfig.securityPolicy.allowPromiscuous = vim.BoolPolicy(value=False)
5680                 dv_pg_spec.defaultPortConfig.securityPolicy.forgedTransmits = vim.BoolPolicy(value=False)
5681                 dv_pg_spec.defaultPortConfig.securityPolicy.macChanges = vim.BoolPolicy(value=False)
5682
5683                 task = dv_switch.AddDVPortgroup_Task([dv_pg_spec])
5684                 self.wait_for_vcenter_task(task, vcenter_conect)
5685
5686                 dvPort_group = self.get_obj(content, [vim.dvs.DistributedVirtualPortgroup], network_name)
5687                 if dvPort_group:
5688                     self.logger.info("Created disributed virtaul port group: {}".format(dvPort_group))
5689                     return dvPort_group.key
5690             else:
5691                 self.logger.debug("No disributed virtual switch found with name {}".format(network_name))
5692
5693         except Exception as exp:
5694             self.logger.error("Error occurred while creating disributed virtaul port group {}"\
5695                              " : {}".format(network_name, exp))
5696         return None
5697
5698     def reconfig_portgroup(self, content, dvPort_group_name , config_info={}):
5699         """
5700          Method to reconfigure disributed virtual portgroup
5701
5702             Args:
5703                 dvPort_group_name - name of disributed virtual portgroup
5704                 content - vCenter content object
5705                 config_info - disributed virtual portgroup configuration
5706
5707             Returns:
5708                 task object
5709         """
5710         try:
5711             dvPort_group = self.get_dvport_group(dvPort_group_name)
5712             if dvPort_group:
5713                 dv_pg_spec = vim.dvs.DistributedVirtualPortgroup.ConfigSpec()
5714                 dv_pg_spec.configVersion = dvPort_group.config.configVersion
5715                 dv_pg_spec.defaultPortConfig = vim.dvs.VmwareDistributedVirtualSwitch.VmwarePortConfigPolicy()
5716                 if "vlanID" in config_info:
5717                     dv_pg_spec.defaultPortConfig.vlan = vim.dvs.VmwareDistributedVirtualSwitch.VlanIdSpec()
5718                     dv_pg_spec.defaultPortConfig.vlan.vlanId = config_info.get('vlanID')
5719
5720                 task = dvPort_group.ReconfigureDVPortgroup_Task(spec=dv_pg_spec)
5721                 return task
5722             else:
5723                 return None
5724         except Exception as exp:
5725             self.logger.error("Error occurred while reconfiguraing disributed virtaul port group {}"\
5726                              " : {}".format(dvPort_group_name, exp))
5727             return None
5728
5729
5730     def destroy_dvport_group(self , dvPort_group_name):
5731         """
5732          Method to destroy disributed virtual portgroup
5733
5734             Args:
5735                 network_name - name of network/portgroup
5736
5737             Returns:
5738                 True if portgroup successfully got deleted else false
5739         """
5740         vcenter_conect, content = self.get_vcenter_content()
5741         try:
5742             status = None
5743             dvPort_group = self.get_dvport_group(dvPort_group_name)
5744             if dvPort_group:
5745                 task = dvPort_group.Destroy_Task()
5746                 status = self.wait_for_vcenter_task(task, vcenter_conect)
5747             return status
5748         except vmodl.MethodFault as exp:
5749             self.logger.error("Caught vmodl fault {} while deleting disributed virtaul port group {}".format(
5750                                                                     exp, dvPort_group_name))
5751             return None
5752
5753
5754     def get_dvport_group(self, dvPort_group_name):
5755         """
5756         Method to get disributed virtual portgroup
5757
5758             Args:
5759                 network_name - name of network/portgroup
5760
5761             Returns:
5762                 portgroup object
5763         """
5764         vcenter_conect, content = self.get_vcenter_content()
5765         dvPort_group = None
5766         try:
5767             container = content.viewManager.CreateContainerView(content.rootFolder, [vim.dvs.DistributedVirtualPortgroup], True)
5768             for item in container.view:
5769                 if item.key == dvPort_group_name:
5770                     dvPort_group = item
5771                     break
5772             return dvPort_group
5773         except vmodl.MethodFault as exp:
5774             self.logger.error("Caught vmodl fault {} for disributed virtaul port group {}".format(
5775                                                                             exp, dvPort_group_name))
5776             return None
5777
5778     def get_vlanID_from_dvs_portgr(self, dvPort_group_name):
5779         """
5780          Method to get disributed virtual portgroup vlanID
5781
5782             Args:
5783                 network_name - name of network/portgroup
5784
5785             Returns:
5786                 vlan ID
5787         """
5788         vlanId = None
5789         try:
5790             dvPort_group = self.get_dvport_group(dvPort_group_name)
5791             if dvPort_group:
5792                 vlanId = dvPort_group.config.defaultPortConfig.vlan.vlanId
5793         except vmodl.MethodFault as exp:
5794             self.logger.error("Caught vmodl fault {} for disributed virtaul port group {}".format(
5795                                                                             exp, dvPort_group_name))
5796         return vlanId
5797
5798
5799     def configure_vlanID(self, content, vcenter_conect, dvPort_group_name):
5800         """
5801          Method to configure vlanID in disributed virtual portgroup vlanID
5802
5803             Args:
5804                 network_name - name of network/portgroup
5805
5806             Returns:
5807                 None
5808         """
5809         vlanID = self.get_vlanID_from_dvs_portgr(dvPort_group_name)
5810         if vlanID == 0:
5811             #configure vlanID
5812             vlanID = self.genrate_vlanID(dvPort_group_name)
5813             config = {"vlanID":vlanID}
5814             task = self.reconfig_portgroup(content, dvPort_group_name,
5815                                     config_info=config)
5816             if task:
5817                 status= self.wait_for_vcenter_task(task, vcenter_conect)
5818                 if status:
5819                     self.logger.info("Reconfigured Port group {} for vlan ID {}".format(
5820                                                         dvPort_group_name,vlanID))
5821             else:
5822                 self.logger.error("Fail reconfigure portgroup {} for vlanID{}".format(
5823                                         dvPort_group_name, vlanID))
5824
5825
5826     def genrate_vlanID(self, network_name):
5827         """
5828          Method to get unused vlanID
5829             Args:
5830                 network_name - name of network/portgroup
5831             Returns:
5832                 vlanID
5833         """
5834         vlan_id = None
5835         used_ids = []
5836         if self.config.get('vlanID_range') == None:
5837             raise vimconn.vimconnConflictException("You must provide a 'vlanID_range' "\
5838                         "at config value before creating sriov network with vlan tag")
5839         if "used_vlanIDs" not in self.persistent_info:
5840                 self.persistent_info["used_vlanIDs"] = {}
5841         else:
5842             used_ids = self.persistent_info["used_vlanIDs"].values()
5843             #For python3
5844             #used_ids = list(self.persistent_info["used_vlanIDs"].values())
5845
5846         for vlanID_range in self.config.get('vlanID_range'):
5847             start_vlanid , end_vlanid = vlanID_range.split("-")
5848             if start_vlanid > end_vlanid:
5849                 raise vimconn.vimconnConflictException("Invalid vlan ID range {}".format(
5850                                                                         vlanID_range))
5851
5852             for id in xrange(int(start_vlanid), int(end_vlanid) + 1):
5853             #For python3
5854             #for id in range(int(start_vlanid), int(end_vlanid) + 1):
5855                 if id not in used_ids:
5856                     vlan_id = id
5857                     self.persistent_info["used_vlanIDs"][network_name] = vlan_id
5858                     return vlan_id
5859         if vlan_id is None:
5860             raise vimconn.vimconnConflictException("All Vlan IDs are in use")
5861
5862
5863     def get_obj(self, content, vimtype, name):
5864         """
5865          Get the vsphere object associated with a given text name
5866         """
5867         obj = None
5868         container = content.viewManager.CreateContainerView(content.rootFolder, vimtype, True)
5869         for item in container.view:
5870             if item.name == name:
5871                 obj = item
5872                 break
5873         return obj
5874
5875
5876     def insert_media_to_vm(self, vapp, image_id):
5877         """
5878         Method to insert media CD-ROM (ISO image) from catalog to vm.
5879         vapp - vapp object to get vm id
5880         Image_id - image id for cdrom to be inerted to vm
5881         """
5882         # create connection object
5883         vca = self.connect()
5884         try:
5885             # fetching catalog details
5886             rest_url = "{}/api/catalog/{}".format(self.url, image_id)
5887             if vca._session:
5888                 headers = {'Accept':'application/*+xml;version=' + API_VERSION,
5889                            'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
5890                 response = self.perform_request(req_type='GET',
5891                                                 url=rest_url,
5892                                                 headers=headers)
5893
5894             if response.status_code != 200:
5895                 self.logger.error("REST call {} failed reason : {}"\
5896                              "status code : {}".format(url_rest_call,
5897                                                     response.content,
5898                                                response.status_code))
5899                 raise vimconn.vimconnException("insert_media_to_vm(): Failed to get "\
5900                                                                     "catalog details")
5901             # searching iso name and id
5902             iso_name,media_id = self.get_media_details(vca, response.content)
5903
5904             if iso_name and media_id:
5905                 data ="""<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
5906                      <ns6:MediaInsertOrEjectParams
5907                      xmlns="http://www.vmware.com/vcloud/versions" xmlns:ns2="http://schemas.dmtf.org/ovf/envelope/1" xmlns:ns3="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData" xmlns:ns4="http://schemas.dmtf.org/wbem/wscim/1/common" xmlns:ns5="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData" xmlns:ns6="http://www.vmware.com/vcloud/v1.5" xmlns:ns7="http://www.vmware.com/schema/ovf" xmlns:ns8="http://schemas.dmtf.org/ovf/environment/1" xmlns:ns9="http://www.vmware.com/vcloud/extension/v1.5">
5908                      <ns6:Media
5909                         type="application/vnd.vmware.vcloud.media+xml"
5910                         name="{}.iso"
5911                         id="urn:vcloud:media:{}"
5912                         href="https://{}/api/media/{}"/>
5913                      </ns6:MediaInsertOrEjectParams>""".format(iso_name, media_id,
5914                                                                 self.url,media_id)
5915
5916                 for vms in vapp.get_all_vms():
5917                     vm_id = vms.get('id').split(':')[-1]
5918
5919                     headers['Content-Type'] = 'application/vnd.vmware.vcloud.mediaInsertOrEjectParams+xml'
5920                     rest_url = "{}/api/vApp/vm-{}/media/action/insertMedia".format(self.url,vm_id)
5921
5922                     response = self.perform_request(req_type='POST',
5923                                                        url=rest_url,
5924                                                           data=data,
5925                                                     headers=headers)
5926
5927                     if response.status_code != 202:
5928                         self.logger.error("Failed to insert CD-ROM to vm")
5929                         raise vimconn.vimconnException("insert_media_to_vm() : Failed to insert"\
5930                                                                                     "ISO image to vm")
5931                     else:
5932                         task = self.get_task_from_response(response.content)
5933                         result = self.client.get_task_monitor().wait_for_success(task=task)
5934                         if result.get('status') == 'success':
5935                             self.logger.info("insert_media_to_vm(): Sucessfully inserted media ISO"\
5936                                                                     " image to vm {}".format(vm_id))
5937
5938         except Exception as exp:
5939             self.logger.error("insert_media_to_vm() : exception occurred "\
5940                                             "while inserting media CD-ROM")
5941             raise vimconn.vimconnException(message=exp)
5942
5943
5944     def get_media_details(self, vca, content):
5945         """
5946         Method to get catalog item details
5947         vca - connection object
5948         content - Catalog details
5949         Return - Media name, media id
5950         """
5951         cataloghref_list = []
5952         try:
5953             if content:
5954                 vm_list_xmlroot = XmlElementTree.fromstring(content)
5955                 for child in vm_list_xmlroot.iter():
5956                     if 'CatalogItem' in child.tag:
5957                         cataloghref_list.append(child.attrib.get('href'))
5958                 if cataloghref_list is not None:
5959                     for href in cataloghref_list:
5960                         if href:
5961                             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
5962                            'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
5963                             response = self.perform_request(req_type='GET',
5964                                                                   url=href,
5965                                                            headers=headers)
5966                             if response.status_code != 200:
5967                                 self.logger.error("REST call {} failed reason : {}"\
5968                                              "status code : {}".format(href,
5969                                                            response.content,
5970                                                       response.status_code))
5971                                 raise vimconn.vimconnException("get_media_details : Failed to get "\
5972                                                                          "catalogitem details")
5973                             list_xmlroot = XmlElementTree.fromstring(response.content)
5974                             for child in list_xmlroot.iter():
5975                                 if 'Entity' in child.tag:
5976                                     if 'media' in child.attrib.get('href'):
5977                                         name = child.attrib.get('name')
5978                                         media_id = child.attrib.get('href').split('/').pop()
5979                                         return name,media_id
5980                             else:
5981                                 self.logger.debug("Media name and id not found")
5982                                 return False,False
5983         except Exception as exp:
5984             self.logger.error("get_media_details : exception occurred "\
5985                                                "getting media details")
5986             raise vimconn.vimconnException(message=exp)
5987
5988
5989     def retry_rest(self, method, url, add_headers=None, data=None):
5990         """ Method to get Token & retry respective REST request
5991             Args:
5992                 api - REST API - Can be one of 'GET' or 'PUT' or 'POST'
5993                 url - request url to be used
5994                 add_headers - Additional headers (optional)
5995                 data - Request payload data to be passed in request
5996             Returns:
5997                 response - Response of request
5998         """
5999         response = None
6000
6001         #Get token
6002         self.get_token()
6003
6004         if self.client._session:
6005                 headers = {'Accept':'application/*+xml;version=' + API_VERSION,
6006                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
6007
6008         if add_headers:
6009             headers.update(add_headers)
6010
6011         if method == 'GET':
6012             response = self.perform_request(req_type='GET',
6013                                             url=url,
6014                                             headers=headers)
6015         elif method == 'PUT':
6016             response = self.perform_request(req_type='PUT',
6017                                             url=url,
6018                                             headers=headers,
6019                                             data=data)
6020         elif method == 'POST':
6021             response = self.perform_request(req_type='POST',
6022                                             url=url,
6023                                             headers=headers,
6024                                             data=data)
6025         elif method == 'DELETE':
6026             response = self.perform_request(req_type='DELETE',
6027                                             url=url,
6028                                             headers=headers)
6029         return response
6030
6031
6032     def get_token(self):
6033         """ Generate a new token if expired
6034
6035             Returns:
6036                 The return client object that letter can be used to connect to vCloud director as admin for VDC
6037         """
6038         try:
6039             self.logger.debug("Generate token for vca {} as {} to datacenter {}.".format(self.org_name,
6040                                                                                       self.user,
6041                                                                                       self.org_name))
6042             host = self.url
6043             client = Client(host, verify_ssl_certs=False)
6044             client.set_credentials(BasicLoginCredentials(self.user, self.org_name, self.passwd))
6045             # connection object
6046             self.client = client
6047
6048         except:
6049             raise vimconn.vimconnConnectionException("Can't connect to a vCloud director org: "
6050                                                      "{} as user: {}".format(self.org_name, self.user))
6051
6052         if not client:
6053             raise vimconn.vimconnConnectionException("Failed while reconnecting vCD")
6054
6055
6056     def get_vdc_details(self):
6057         """ Get VDC details using pyVcloud Lib
6058
6059             Returns org and vdc object
6060         """
6061         org = Org(self.client, resource=self.client.get_org())
6062         vdc = org.get_vdc(self.tenant_name)
6063
6064         #Retry once, if failed by refreshing token
6065         if vdc is None:
6066             self.get_token()
6067             vdc = org.get_vdc(self.tenant_name)
6068
6069         return org, vdc
6070
6071
6072     def perform_request(self, req_type, url, headers=None, data=None):
6073         """Perform the POST/PUT/GET/DELETE request."""
6074
6075         #Log REST request details
6076         self.log_request(req_type, url=url, headers=headers, data=data)
6077         # perform request and return its result
6078         if req_type == 'GET':
6079             response = requests.get(url=url,
6080                                 headers=headers,
6081                                 verify=False)
6082         elif req_type == 'PUT':
6083             response = requests.put(url=url,
6084                                 headers=headers,
6085                                 data=data,
6086                                 verify=False)
6087         elif req_type == 'POST':
6088             response = requests.post(url=url,
6089                                  headers=headers,
6090                                  data=data,
6091                                  verify=False)
6092         elif req_type == 'DELETE':
6093             response = requests.delete(url=url,
6094                                  headers=headers,
6095                                  verify=False)
6096         #Log the REST response
6097         self.log_response(response)
6098
6099         return response
6100
6101
6102     def log_request(self, req_type, url=None, headers=None, data=None):
6103         """Logs REST request details"""
6104
6105         if req_type is not None:
6106             self.logger.debug("Request type: {}".format(req_type))
6107
6108         if url is not None:
6109             self.logger.debug("Request url: {}".format(url))
6110
6111         if headers is not None:
6112             for header in headers:
6113                 self.logger.debug("Request header: {}: {}".format(header, headers[header]))
6114
6115         if data is not None:
6116             self.logger.debug("Request data: {}".format(data))
6117
6118
6119     def log_response(self, response):
6120         """Logs REST response details"""
6121
6122         self.logger.debug("Response status code: {} ".format(response.status_code))
6123
6124
6125     def get_task_from_response(self, content):
6126         """
6127         content - API response content(response.content)
6128         return task object
6129         """
6130         xmlroot = XmlElementTree.fromstring(content)
6131         if xmlroot.tag.split('}')[1] == "Task":
6132             return xmlroot
6133         else:
6134             for ele in xmlroot:
6135                 if ele.tag.split("}")[1] == "Tasks":
6136                     task = ele[0]
6137                     break
6138             return task
6139
6140
6141     def power_on_vapp(self,vapp_id, vapp_name):
6142         """
6143         vapp_id - vApp uuid
6144         vapp_name - vAapp name
6145         return - Task object
6146         """
6147         headers = {'Accept':'application/*+xml;version=' + API_VERSION,
6148                    'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
6149
6150         poweron_href = "{}/api/vApp/vapp-{}/power/action/powerOn".format(self.url,
6151                                                                           vapp_id)
6152         response = self.perform_request(req_type='POST',
6153                                        url=poweron_href,
6154                                         headers=headers)
6155
6156         if response.status_code != 202:
6157             self.logger.error("REST call {} failed reason : {}"\
6158                          "status code : {} ".format(poweron_href,
6159                                                 response.content,
6160                                            response.status_code))
6161             raise vimconn.vimconnException("power_on_vapp() : Failed to power on "\
6162                                                       "vApp {}".format(vapp_name))
6163         else:
6164             poweron_task = self.get_task_from_response(response.content)
6165             return poweron_task
6166
6167