osm_ro/vimconn_vmware.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2016-2017 VMware Inc.
   5 # This file is part of ETSI OSM
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact:  osslegalrouting@vmware.com
  22 ##
  23
  24 """
  25 vimconn_vmware implementation an Abstract class in order to interact with VMware  vCloud Director.
  26 mbayramov@vmware.com
  27 """
  28 from progressbar import Percentage, Bar, ETA, FileTransferSpeed, ProgressBar
  29
  30 import vimconn
  31 import os
  32 import traceback
  33 import itertools
  34 import requests
  35 import ssl
  36 import atexit
  37
  38 from pyVmomi import vim, vmodl
  39 from pyVim.connect import SmartConnect, Disconnect
  40
  41 from xml.etree import ElementTree as XmlElementTree
  42 from lxml import etree as lxmlElementTree
  43
  44 import yaml
  45 from pyvcloud.vcd.client import BasicLoginCredentials,Client,VcdTaskException
  46 from pyvcloud.vcd.vdc import VDC
  47 from pyvcloud.vcd.org import Org
  48 import re
  49 from pyvcloud.vcd.vapp import VApp
  50 from xml.sax.saxutils import escape
  51 import logging
  52 import json
  53 import time
  54 import uuid
  55 import httplib
  56 #For python3
  57 #import http.client
  58 import hashlib
  59 import socket
  60 import struct
  61 import netaddr
  62 import random
  63
  64 # global variable for vcd connector type
  65 STANDALONE = 'standalone'
  66
  67 # key for flavor dicts
  68 FLAVOR_RAM_KEY = 'ram'
  69 FLAVOR_VCPUS_KEY = 'vcpus'
  70 FLAVOR_DISK_KEY = 'disk'
  71 DEFAULT_IP_PROFILE = {'dhcp_count':50,
  72                       'dhcp_enabled':True,
  73                       'ip_version':"IPv4"
  74                       }
  75 # global variable for wait time
  76 INTERVAL_TIME = 5
  77 MAX_WAIT_TIME = 1800
  78
  79 API_VERSION = '5.9'
  80
  81 __author__ = "Mustafa Bayramov, Arpita Kate, Sachin Bhangare, Prakash Kasar"
  82 __date__ = "$09-Mar-2018 11:09:29$"
  83 __version__ = '0.2'
  84
  85 #     -1: "Could not be created",
  86 #     0: "Unresolved",
  87 #     1: "Resolved",
  88 #     2: "Deployed",
  89 #     3: "Suspended",
  90 #     4: "Powered on",
  91 #     5: "Waiting for user input",
  92 #     6: "Unknown state",
  93 #     7: "Unrecognized state",
  94 #     8: "Powered off",
  95 #     9: "Inconsistent state",
  96 #     10: "Children do not all have the same status",
  97 #     11: "Upload initiated, OVF descriptor pending",
  98 #     12: "Upload initiated, copying contents",
  99 #     13: "Upload initiated , disk contents pending",
 100 #     14: "Upload has been quarantined",
 101 #     15: "Upload quarantine period has expired"
 102
 103 # mapping vCD status to MANO
 104 vcdStatusCode2manoFormat = {4: 'ACTIVE',
 105                             7: 'PAUSED',
 106                             3: 'SUSPENDED',
 107                             8: 'INACTIVE',
 108                             12: 'BUILD',
 109                             -1: 'ERROR',
 110                             14: 'DELETED'}
 111
 112 #
 113 netStatus2manoFormat = {'ACTIVE': 'ACTIVE', 'PAUSED': 'PAUSED', 'INACTIVE': 'INACTIVE', 'BUILD': 'BUILD',
 114                         'ERROR': 'ERROR', 'DELETED': 'DELETED'
 115                         }
 116
 117 class vimconnector(vimconn.vimconnector):
 118     # dict used to store flavor in memory
 119     flavorlist = {}
 120
 121     def __init__(self, uuid=None, name=None, tenant_id=None, tenant_name=None,
 122                  url=None, url_admin=None, user=None, passwd=None, log_level=None, config={}, persistent_info={}):
 123         """
 124         Constructor create vmware connector to vCloud director.
 125
 126         By default construct doesn't validate connection state. So client can create object with None arguments.
 127         If client specified username , password and host and VDC name.  Connector initialize other missing attributes.
 128
 129         a) It initialize organization UUID
 130         b) Initialize tenant_id/vdc ID.   (This information derived from tenant name)
 131
 132         Args:
 133             uuid - is organization uuid.
 134             name - is organization name that must be presented in vCloud director.
 135             tenant_id - is VDC uuid it must be presented in vCloud director
 136             tenant_name - is VDC name.
 137             url - is hostname or ip address of vCloud director
 138             url_admin - same as above.
 139             user - is user that administrator for organization. Caller must make sure that
 140                     username has right privileges.
 141
 142             password - is password for a user.
 143
 144             VMware connector also requires PVDC administrative privileges and separate account.
 145             This variables must be passed via config argument dict contains keys
 146
 147             dict['admin_username']
 148             dict['admin_password']
 149             config - Provide NSX and vCenter information
 150
 151             Returns:
 152                 Nothing.
 153         """
 154
 155         vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url,
 156                                       url_admin, user, passwd, log_level, config)
 157
 158         self.logger = logging.getLogger('openmano.vim.vmware')
 159         self.logger.setLevel(10)
 160         self.persistent_info = persistent_info
 161
 162         self.name = name
 163         self.id = uuid
 164         self.url = url
 165         self.url_admin = url_admin
 166         self.tenant_id = tenant_id
 167         self.tenant_name = tenant_name
 168         self.user = user
 169         self.passwd = passwd
 170         self.config = config
 171         self.admin_password = None
 172         self.admin_user = None
 173         self.org_name = ""
 174         self.nsx_manager = None
 175         self.nsx_user = None
 176         self.nsx_password = None
 177         self.availability_zone = None
 178
 179         # Disable warnings from self-signed certificates.
 180         requests.packages.urllib3.disable_warnings()
 181
 182         if tenant_name is not None:
 183             orgnameandtenant = tenant_name.split(":")
 184             if len(orgnameandtenant) == 2:
 185                 self.tenant_name = orgnameandtenant[1]
 186                 self.org_name = orgnameandtenant[0]
 187             else:
 188                 self.tenant_name = tenant_name
 189         if "orgname" in config:
 190             self.org_name = config['orgname']
 191
 192         if log_level:
 193             self.logger.setLevel(getattr(logging, log_level))
 194
 195         try:
 196             self.admin_user = config['admin_username']
 197             self.admin_password = config['admin_password']
 198         except KeyError:
 199             raise vimconn.vimconnException(message="Error admin username or admin password is empty.")
 200
 201         try:
 202             self.nsx_manager = config['nsx_manager']
 203             self.nsx_user = config['nsx_user']
 204             self.nsx_password = config['nsx_password']
 205         except KeyError:
 206             raise vimconn.vimconnException(message="Error: nsx manager or nsx user or nsx password is empty in Config")
 207
 208         self.vcenter_ip = config.get("vcenter_ip", None)
 209         self.vcenter_port = config.get("vcenter_port", None)
 210         self.vcenter_user = config.get("vcenter_user", None)
 211         self.vcenter_password = config.get("vcenter_password", None)
 212
 213         #Set availability zone for Affinity rules
 214         self.availability_zone = self.set_availability_zones()
 215
 216 # ############# Stub code for SRIOV #################
 217 #         try:
 218 #             self.dvs_name = config['dv_switch_name']
 219 #         except KeyError:
 220 #             raise vimconn.vimconnException(message="Error: distributed virtaul switch name is empty in Config")
 221 #
 222 #         self.vlanID_range = config.get("vlanID_range", None)
 223
 224         self.org_uuid = None
 225         self.client = None
 226
 227         if not url:
 228             raise vimconn.vimconnException('url param can not be NoneType')
 229
 230         if not self.url_admin:  # try to use normal url
 231             self.url_admin = self.url
 232
 233         logging.debug("UUID: {} name: {} tenant_id: {} tenant name {}".format(self.id, self.org_name,
 234                                                                               self.tenant_id, self.tenant_name))
 235         logging.debug("vcd url {} vcd username: {} vcd password: {}".format(self.url, self.user, self.passwd))
 236         logging.debug("vcd admin username {} vcd admin passowrd {}".format(self.admin_user, self.admin_password))
 237
 238         # initialize organization
 239         if self.user is not None and self.passwd is not None and self.url:
 240             self.init_organization()
 241
 242     def __getitem__(self, index):
 243         if index == 'name':
 244             return self.name
 245         if index == 'tenant_id':
 246             return self.tenant_id
 247         if index == 'tenant_name':
 248             return self.tenant_name
 249         elif index == 'id':
 250             return self.id
 251         elif index == 'org_name':
 252             return self.org_name
 253         elif index == 'org_uuid':
 254             return self.org_uuid
 255         elif index == 'user':
 256             return self.user
 257         elif index == 'passwd':
 258             return self.passwd
 259         elif index == 'url':
 260             return self.url
 261         elif index == 'url_admin':
 262             return self.url_admin
 263         elif index == "config":
 264             return self.config
 265         else:
 266             raise KeyError("Invalid key '%s'" % str(index))
 267
 268     def __setitem__(self, index, value):
 269         if index == 'name':
 270             self.name = value
 271         if index == 'tenant_id':
 272             self.tenant_id = value
 273         if index == 'tenant_name':
 274             self.tenant_name = value
 275         elif index == 'id':
 276             self.id = value
 277         elif index == 'org_name':
 278             self.org_name = value
 279         elif index == 'org_uuid':
 280             self.org_uuid = value
 281         elif index == 'user':
 282             self.user = value
 283         elif index == 'passwd':
 284             self.passwd = value
 285         elif index == 'url':
 286             self.url = value
 287         elif index == 'url_admin':
 288             self.url_admin = value
 289         else:
 290             raise KeyError("Invalid key '%s'" % str(index))
 291
 292     def connect_as_admin(self):
 293         """ Method connect as pvdc admin user to vCloud director.
 294             There are certain action that can be done only by provider vdc admin user.
 295             Organization creation / provider network creation etc.
 296
 297             Returns:
 298                 The return client object that latter can be used to connect to vcloud director as admin for provider vdc
 299         """
 300
 301         self.logger.debug("Logging into vCD {} as admin.".format(self.org_name))
 302
 303         try:
 304             host = self.url
 305             org = 'System'
 306             client_as_admin = Client(host, verify_ssl_certs=False)
 307             client_as_admin.set_credentials(BasicLoginCredentials(self.admin_user, org, self.admin_password))
 308         except Exception as e:
 309             raise vimconn.vimconnException(
 310                   "Can't connect to a vCloud director as: {} with exception {}".format(self.admin_user, e))
 311
 312         return client_as_admin
 313
 314     def connect(self):
 315         """ Method connect as normal user to vCloud director.
 316
 317             Returns:
 318                 The return client object that latter can be used to connect to vCloud director as admin for VDC
 319         """
 320
 321         try:
 322             self.logger.debug("Logging into vCD {} as {} to datacenter {}.".format(self.org_name,
 323                                                                                       self.user,
 324                                                                                       self.org_name))
 325             host = self.url
 326             client = Client(host, verify_ssl_certs=False)
 327             client.set_credentials(BasicLoginCredentials(self.user, self.org_name, self.passwd))
 328         except:
 329             raise vimconn.vimconnConnectionException("Can't connect to a vCloud director org: "
 330                                                      "{} as user: {}".format(self.org_name, self.user))
 331
 332         return client
 333
 334     def init_organization(self):
 335         """ Method initialize organization UUID and VDC parameters.
 336
 337             At bare minimum client must provide organization name that present in vCloud director and VDC.
 338
 339             The VDC - UUID ( tenant_id) will be initialized at the run time if client didn't call constructor.
 340             The Org - UUID will be initialized at the run time if data center present in vCloud director.
 341
 342             Returns:
 343                 The return vca object that letter can be used to connect to vcloud direct as admin
 344         """
 345         client = self.connect()
 346         if not client:
 347             raise vimconn.vimconnConnectionException("Failed to connect vCD.")
 348
 349         self.client = client
 350         try:
 351             if self.org_uuid is None:
 352                 org_list = client.get_org_list()
 353                 for org in org_list.Org:
 354                     # we set org UUID at the init phase but we can do it only when we have valid credential.
 355                     if org.get('name') == self.org_name:
 356                         self.org_uuid = org.get('href').split('/')[-1]
 357                         self.logger.debug("Setting organization UUID {}".format(self.org_uuid))
 358                         break
 359                 else:
 360                     raise vimconn.vimconnException("Vcloud director organization {} not found".format(self.org_name))
 361
 362                 # if well good we require for org details
 363                 org_details_dict = self.get_org(org_uuid=self.org_uuid)
 364
 365                 # we have two case if we want to initialize VDC ID or VDC name at run time
 366                 # tenant_name provided but no tenant id
 367                 if self.tenant_id is None and self.tenant_name is not None and 'vdcs' in org_details_dict:
 368                     vdcs_dict = org_details_dict['vdcs']
 369                     for vdc in vdcs_dict:
 370                         if vdcs_dict[vdc] == self.tenant_name:
 371                             self.tenant_id = vdc
 372                             self.logger.debug("Setting vdc uuid {} for organization UUID {}".format(self.tenant_id,
 373                                                                                                     self.org_name))
 374                             break
 375                     else:
 376                         raise vimconn.vimconnException("Tenant name indicated but not present in vcloud director.")
 377                     # case two we have tenant_id but we don't have tenant name so we find and set it.
 378                     if self.tenant_id is not None and self.tenant_name is None and 'vdcs' in org_details_dict:
 379                         vdcs_dict = org_details_dict['vdcs']
 380                         for vdc in vdcs_dict:
 381                             if vdc == self.tenant_id:
 382                                 self.tenant_name = vdcs_dict[vdc]
 383                                 self.logger.debug("Setting vdc uuid {} for organization UUID {}".format(self.tenant_id,
 384                                                                                                         self.org_name))
 385                                 break
 386                         else:
 387                             raise vimconn.vimconnException("Tenant id indicated but not present in vcloud director")
 388             self.logger.debug("Setting organization uuid {}".format(self.org_uuid))
 389         except:
 390             self.logger.debug("Failed initialize organization UUID for org {}".format(self.org_name))
 391             self.logger.debug(traceback.format_exc())
 392             self.org_uuid = None
 393
 394     def new_tenant(self, tenant_name=None, tenant_description=None):
 395         """ Method adds a new tenant to VIM with this name.
 396             This action requires access to create VDC action in vCloud director.
 397
 398             Args:
 399                 tenant_name is tenant_name to be created.
 400                 tenant_description not used for this call
 401
 402             Return:
 403                 returns the tenant identifier in UUID format.
 404                 If action is failed method will throw vimconn.vimconnException method
 405             """
 406         vdc_task = self.create_vdc(vdc_name=tenant_name)
 407         if vdc_task is not None:
 408             vdc_uuid, value = vdc_task.popitem()
 409             self.logger.info("Created new vdc {} and uuid: {}".format(tenant_name, vdc_uuid))
 410             return vdc_uuid
 411         else:
 412             raise vimconn.vimconnException("Failed create tenant {}".format(tenant_name))
 413
 414     def delete_tenant(self, tenant_id=None):
 415         """ Delete a tenant from VIM
 416              Args:
 417                 tenant_id is tenant_id to be deleted.
 418
 419             Return:
 420                 returns the tenant identifier in UUID format.
 421                 If action is failed method will throw exception
 422         """
 423         vca = self.connect_as_admin()
 424         if not vca:
 425             raise vimconn.vimconnConnectionException("Failed to connect vCD")
 426
 427         if tenant_id is not None:
 428             if vca._session:
 429                 #Get OrgVDC
 430                 url_list = [self.url, '/api/vdc/', tenant_id]
 431                 orgvdc_herf = ''.join(url_list)
 432
 433                 headers = {'Accept':'application/*+xml;version=' + API_VERSION,
 434                            'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
 435                 response = self.perform_request(req_type='GET',
 436                                                 url=orgvdc_herf,
 437                                                 headers=headers)
 438
 439                 if response.status_code != requests.codes.ok:
 440                     self.logger.debug("delete_tenant():GET REST API call {} failed. "\
 441                                       "Return status code {}".format(orgvdc_herf,
 442                                                                      response.status_code))
 443                     raise vimconn.vimconnNotFoundException("Fail to get tenant {}".format(tenant_id))
 444
 445                 lxmlroot_respond = lxmlElementTree.fromstring(response.content)
 446                 namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.iteritems() if prefix}
 447                 #For python3
 448                 #namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.items() if prefix}
 449                 namespaces["xmlns"]= "http://www.vmware.com/vcloud/v1.5"
 450                 vdc_remove_href = lxmlroot_respond.find("xmlns:Link[@rel='remove']",namespaces).attrib['href']
 451                 vdc_remove_href = vdc_remove_href + '?recursive=true&force=true'
 452
 453                 response = self.perform_request(req_type='DELETE',
 454                                                 url=vdc_remove_href,
 455                                                 headers=headers)
 456
 457                 if response.status_code == 202:
 458                     time.sleep(5)
 459                     return tenant_id
 460                 else:
 461                     self.logger.debug("delete_tenant(): DELETE REST API call {} failed. "\
 462                                       "Return status code {}".format(vdc_remove_href,
 463                                                                      response.status_code))
 464                     raise vimconn.vimconnException("Fail to delete tenant with ID {}".format(tenant_id))
 465         else:
 466             self.logger.debug("delete_tenant():Incorrect tenant ID  {}".format(tenant_id))
 467             raise vimconn.vimconnNotFoundException("Fail to get tenant {}".format(tenant_id))
 468
 469
 470     def get_tenant_list(self, filter_dict={}):
 471         """Obtain tenants of VIM
 472         filter_dict can contain the following keys:
 473             name: filter by tenant name
 474             id: filter by tenant uuid/id
 475             <other VIM specific>
 476         Returns the tenant list of dictionaries:
 477             [{'name':'<name>, 'id':'<id>, ...}, ...]
 478
 479         """
 480         org_dict = self.get_org(self.org_uuid)
 481         vdcs_dict = org_dict['vdcs']
 482
 483         vdclist = []
 484         try:
 485             for k in vdcs_dict:
 486                 entry = {'name': vdcs_dict[k], 'id': k}
 487                 # if caller didn't specify dictionary we return all tenants.
 488                 if filter_dict is not None and filter_dict:
 489                     filtered_entry = entry.copy()
 490                     filtered_dict = set(entry.keys()) - set(filter_dict)
 491                     for unwanted_key in filtered_dict: del entry[unwanted_key]
 492                     if filter_dict == entry:
 493                         vdclist.append(filtered_entry)
 494                 else:
 495                     vdclist.append(entry)
 496         except:
 497             self.logger.debug("Error in get_tenant_list()")
 498             self.logger.debug(traceback.format_exc())
 499             raise vimconn.vimconnException("Incorrect state. {}")
 500
 501         return vdclist
 502
 503     def new_network(self, net_name, net_type, ip_profile=None, shared=False):
 504         """Adds a tenant network to VIM
 505             net_name is the name
 506             net_type can be 'bridge','data'.'ptp'.
 507             ip_profile is a dict containing the IP parameters of the network
 508             shared is a boolean
 509         Returns the network identifier"""
 510
 511         self.logger.debug("new_network tenant {} net_type {} ip_profile {} shared {}"
 512                           .format(net_name, net_type, ip_profile, shared))
 513
 514         isshared = 'false'
 515         if shared:
 516             isshared = 'true'
 517
 518 # ############# Stub code for SRIOV #################
 519 #         if net_type == "data" or net_type == "ptp":
 520 #             if self.config.get('dv_switch_name') == None:
 521 #                  raise vimconn.vimconnConflictException("You must provide 'dv_switch_name' at config value")
 522 #             network_uuid = self.create_dvPort_group(net_name)
 523
 524         network_uuid = self.create_network(network_name=net_name, net_type=net_type,
 525                                            ip_profile=ip_profile, isshared=isshared)
 526         if network_uuid is not None:
 527             return network_uuid
 528         else:
 529             raise vimconn.vimconnUnexpectedResponse("Failed create a new network {}".format(net_name))
 530
 531     def get_vcd_network_list(self):
 532         """ Method available organization for a logged in tenant
 533
 534             Returns:
 535                 The return vca object that letter can be used to connect to vcloud direct as admin
 536         """
 537
 538         self.logger.debug("get_vcd_network_list(): retrieving network list for vcd {}".format(self.tenant_name))
 539
 540         if not self.tenant_name:
 541             raise vimconn.vimconnConnectionException("Tenant name is empty.")
 542
 543         org, vdc = self.get_vdc_details()
 544         if vdc is None:
 545             raise vimconn.vimconnConnectionException("Can't retrieve information for a VDC {}".format(self.tenant_name))
 546
 547         vdc_uuid = vdc.get('id').split(":")[3]
 548         if self.client._session:
 549                 headers = {'Accept':'application/*+xml;version=' + API_VERSION,
 550                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
 551                 response = self.perform_request(req_type='GET',
 552                                            url=vdc.get('href'),
 553                                                headers=headers)
 554         if response.status_code != 200:
 555             self.logger.error("Failed to get vdc content")
 556             raise vimconn.vimconnNotFoundException("Failed to get vdc content")
 557         else:
 558             content = XmlElementTree.fromstring(response.content)
 559
 560         network_list = []
 561         try:
 562             for item in content:
 563                 if item.tag.split('}')[-1] == 'AvailableNetworks':
 564                     for net in item:
 565                         response = self.perform_request(req_type='GET',
 566                                                    url=net.get('href'),
 567                                                        headers=headers)
 568
 569                         if response.status_code != 200:
 570                             self.logger.error("Failed to get network content")
 571                             raise vimconn.vimconnNotFoundException("Failed to get network content")
 572                         else:
 573                             net_details = XmlElementTree.fromstring(response.content)
 574
 575                             filter_dict = {}
 576                             net_uuid = net_details.get('id').split(":")
 577                             if len(net_uuid) != 4:
 578                                 continue
 579                             else:
 580                                 net_uuid = net_uuid[3]
 581                                 # create dict entry
 582                                 self.logger.debug("Adding  {} to a list vcd id {} network {}".format(net_uuid,
 583                                                                                                         vdc_uuid,
 584                                                                                          net_details.get('name')))
 585                                 filter_dict["name"] = net_details.get('name')
 586                                 filter_dict["id"] = net_uuid
 587                                 if [i.text for i in net_details if i.tag.split('}')[-1] == 'IsShared'][0] == 'true':
 588                                     shared = True
 589                                 else:
 590                                     shared = False
 591                                 filter_dict["shared"] = shared
 592                                 filter_dict["tenant_id"] = vdc_uuid
 593                                 if int(net_details.get('status')) == 1:
 594                                     filter_dict["admin_state_up"] = True
 595                                 else:
 596                                     filter_dict["admin_state_up"] = False
 597                                 filter_dict["status"] = "ACTIVE"
 598                                 filter_dict["type"] = "bridge"
 599                                 network_list.append(filter_dict)
 600                                 self.logger.debug("get_vcd_network_list adding entry {}".format(filter_dict))
 601         except:
 602             self.logger.debug("Error in get_vcd_network_list", exc_info=True)
 603             pass
 604
 605         self.logger.debug("get_vcd_network_list returning {}".format(network_list))
 606         return network_list
 607
 608     def get_network_list(self, filter_dict={}):
 609         """Obtain tenant networks of VIM
 610         Filter_dict can be:
 611             name: network name  OR/AND
 612             id: network uuid    OR/AND
 613             shared: boolean     OR/AND
 614             tenant_id: tenant   OR/AND
 615             admin_state_up: boolean
 616             status: 'ACTIVE'
 617
 618         [{key : value , key : value}]
 619
 620         Returns the network list of dictionaries:
 621             [{<the fields at Filter_dict plus some VIM specific>}, ...]
 622             List can be empty
 623         """
 624
 625         self.logger.debug("get_network_list(): retrieving network list for vcd {}".format(self.tenant_name))
 626
 627         if not self.tenant_name:
 628             raise vimconn.vimconnConnectionException("Tenant name is empty.")
 629
 630         org, vdc = self.get_vdc_details()
 631         if vdc is None:
 632             raise vimconn.vimconnConnectionException("Can't retrieve information for a VDC {}.".format(self.tenant_name))
 633
 634         try:
 635             vdcid = vdc.get('id').split(":")[3]
 636
 637             if self.client._session:
 638                 headers = {'Accept':'application/*+xml;version=' + API_VERSION,
 639                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
 640                 response = self.perform_request(req_type='GET',
 641                                            url=vdc.get('href'),
 642                                                headers=headers)
 643             if response.status_code != 200:
 644                 self.logger.error("Failed to get vdc content")
 645                 raise vimconn.vimconnNotFoundException("Failed to get vdc content")
 646             else:
 647                 content = XmlElementTree.fromstring(response.content)
 648
 649             network_list = []
 650             for item in content:
 651                 if item.tag.split('}')[-1] == 'AvailableNetworks':
 652                     for net in item:
 653                         response = self.perform_request(req_type='GET',
 654                                                    url=net.get('href'),
 655                                                        headers=headers)
 656
 657                         if response.status_code != 200:
 658                             self.logger.error("Failed to get network content")
 659                             raise vimconn.vimconnNotFoundException("Failed to get network content")
 660                         else:
 661                             net_details = XmlElementTree.fromstring(response.content)
 662
 663                             filter_entry = {}
 664                             net_uuid = net_details.get('id').split(":")
 665                             if len(net_uuid) != 4:
 666                                 continue
 667                             else:
 668                                 net_uuid = net_uuid[3]
 669                                 # create dict entry
 670                                 self.logger.debug("Adding  {} to a list vcd id {} network {}".format(net_uuid,
 671                                                                                                         vdcid,
 672                                                                                          net_details.get('name')))
 673                                 filter_entry["name"] = net_details.get('name')
 674                                 filter_entry["id"] = net_uuid
 675                                 if [i.text for i in net_details if i.tag.split('}')[-1] == 'IsShared'][0] == 'true':
 676                                     shared = True
 677                                 else:
 678                                     shared = False
 679                                 filter_entry["shared"] = shared
 680                                 filter_entry["tenant_id"] = vdcid
 681                                 if int(net_details.get('status')) == 1:
 682                                     filter_entry["admin_state_up"] = True
 683                                 else:
 684                                     filter_entry["admin_state_up"] = False
 685                                 filter_entry["status"] = "ACTIVE"
 686                                 filter_entry["type"] = "bridge"
 687                                 filtered_entry = filter_entry.copy()
 688
 689                                 if filter_dict is not None and filter_dict:
 690                                     # we remove all the key : value we don't care and match only
 691                                     # respected field
 692                                     filtered_dict = set(filter_entry.keys()) - set(filter_dict)
 693                                     for unwanted_key in filtered_dict: del filter_entry[unwanted_key]
 694                                     if filter_dict == filter_entry:
 695                                         network_list.append(filtered_entry)
 696                                 else:
 697                                     network_list.append(filtered_entry)
 698         except Exception as e:
 699             self.logger.debug("Error in get_network_list",exc_info=True)
 700             if isinstance(e, vimconn.vimconnException):
 701                 raise
 702             else:
 703                 raise vimconn.vimconnNotFoundException("Failed : Networks list not found {} ".format(e))
 704
 705         self.logger.debug("Returning {}".format(network_list))
 706         return network_list
 707
 708     def get_network(self, net_id):
 709         """Method obtains network details of net_id VIM network
 710            Return a dict with  the fields at filter_dict (see get_network_list) plus some VIM specific>}, ...]"""
 711
 712         try:
 713             org, vdc = self.get_vdc_details()
 714             vdc_id = vdc.get('id').split(":")[3]
 715             if self.client._session:
 716                 headers = {'Accept':'application/*+xml;version=' + API_VERSION,
 717                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
 718                 response = self.perform_request(req_type='GET',
 719                                            url=vdc.get('href'),
 720                                                headers=headers)
 721             if response.status_code != 200:
 722                 self.logger.error("Failed to get vdc content")
 723                 raise vimconn.vimconnNotFoundException("Failed to get vdc content")
 724             else:
 725                 content = XmlElementTree.fromstring(response.content)
 726
 727             filter_dict = {}
 728
 729             for item in content:
 730                 if item.tag.split('}')[-1] == 'AvailableNetworks':
 731                     for net in item:
 732                         response = self.perform_request(req_type='GET',
 733                                                    url=net.get('href'),
 734                                                        headers=headers)
 735
 736                         if response.status_code != 200:
 737                             self.logger.error("Failed to get network content")
 738                             raise vimconn.vimconnNotFoundException("Failed to get network content")
 739                         else:
 740                             net_details = XmlElementTree.fromstring(response.content)
 741
 742                             vdc_network_id = net_details.get('id').split(":")
 743                             if len(vdc_network_id) == 4 and vdc_network_id[3] == net_id:
 744                                 filter_dict["name"] = net_details.get('name')
 745                                 filter_dict["id"] = vdc_network_id[3]
 746                                 if [i.text for i in net_details if i.tag.split('}')[-1] == 'IsShared'][0] == 'true':
 747                                     shared = True
 748                                 else:
 749                                     shared = False
 750                                 filter_dict["shared"] = shared
 751                                 filter_dict["tenant_id"] = vdc_id
 752                                 if int(net_details.get('status')) == 1:
 753                                     filter_dict["admin_state_up"] = True
 754                                 else:
 755                                     filter_dict["admin_state_up"] = False
 756                                 filter_dict["status"] = "ACTIVE"
 757                                 filter_dict["type"] = "bridge"
 758                                 self.logger.debug("Returning {}".format(filter_dict))
 759                                 return filter_dict
 760                     else:
 761                         raise vimconn.vimconnNotFoundException("Network {} not found".format(net_id))
 762         except Exception as e:
 763             self.logger.debug("Error in get_network")
 764             self.logger.debug(traceback.format_exc())
 765             if isinstance(e, vimconn.vimconnException):
 766                 raise
 767             else:
 768                 raise vimconn.vimconnNotFoundException("Failed : Network not found {} ".format(e))
 769
 770         return filter_dict
 771
 772     def delete_network(self, net_id):
 773         """
 774             Method Deletes a tenant network from VIM, provide the network id.
 775
 776             Returns the network identifier or raise an exception
 777         """
 778
 779         # ############# Stub code for SRIOV #################
 780 #         dvport_group = self.get_dvport_group(net_id)
 781 #         if dvport_group:
 782 #             #delete portgroup
 783 #             status = self.destroy_dvport_group(net_id)
 784 #             if status:
 785 #                 # Remove vlanID from persistent info
 786 #                 if net_id in self.persistent_info["used_vlanIDs"]:
 787 #                     del self.persistent_info["used_vlanIDs"][net_id]
 788 #
 789 #                 return net_id
 790
 791         vcd_network = self.get_vcd_network(network_uuid=net_id)
 792         if vcd_network is not None and vcd_network:
 793             if self.delete_network_action(network_uuid=net_id):
 794                 return net_id
 795         else:
 796             raise vimconn.vimconnNotFoundException("Network {} not found".format(net_id))
 797
 798     def refresh_nets_status(self, net_list):
 799         """Get the status of the networks
 800            Params: the list of network identifiers
 801            Returns a dictionary with:
 802                 net_id:         #VIM id of this network
 803                     status:     #Mandatory. Text with one of:
 804                                 #  DELETED (not found at vim)
 805                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 806                                 #  OTHER (Vim reported other status not understood)
 807                                 #  ERROR (VIM indicates an ERROR status)
 808                                 #  ACTIVE, INACTIVE, DOWN (admin down),
 809                                 #  BUILD (on building process)
 810                                 #
 811                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 812                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 813
 814         """
 815
 816         dict_entry = {}
 817         try:
 818             for net in net_list:
 819                 errormsg = ''
 820                 vcd_network = self.get_vcd_network(network_uuid=net)
 821                 if vcd_network is not None and vcd_network:
 822                     if vcd_network['status'] == '1':
 823                         status = 'ACTIVE'
 824                     else:
 825                         status = 'DOWN'
 826                 else:
 827                     status = 'DELETED'
 828                     errormsg = 'Network not found.'
 829
 830                 dict_entry[net] = {'status': status, 'error_msg': errormsg,
 831                                    'vim_info': yaml.safe_dump(vcd_network)}
 832         except:
 833             self.logger.debug("Error in refresh_nets_status")
 834             self.logger.debug(traceback.format_exc())
 835
 836         return dict_entry
 837
 838     def get_flavor(self, flavor_id):
 839         """Obtain flavor details from the  VIM
 840             Returns the flavor dict details {'id':<>, 'name':<>, other vim specific } #TODO to concrete
 841         """
 842         if flavor_id not in vimconnector.flavorlist:
 843             raise vimconn.vimconnNotFoundException("Flavor not found.")
 844         return vimconnector.flavorlist[flavor_id]
 845
 846     def new_flavor(self, flavor_data):
 847         """Adds a tenant flavor to VIM
 848             flavor_data contains a dictionary with information, keys:
 849                 name: flavor name
 850                 ram: memory (cloud type) in MBytes
 851                 vpcus: cpus (cloud type)
 852                 extended: EPA parameters
 853                   - numas: #items requested in same NUMA
 854                         memory: number of 1G huge pages memory
 855                         paired-threads|cores|threads: number of paired hyperthreads, complete cores OR individual threads
 856                         interfaces: # passthrough(PT) or SRIOV interfaces attached to this numa
 857                           - name: interface name
 858                             dedicated: yes|no|yes:sriov;  for PT, SRIOV or only one SRIOV for the physical NIC
 859                             bandwidth: X Gbps; requested guarantee bandwidth
 860                             vpci: requested virtual PCI address
 861                 disk: disk size
 862                 is_public:
 863                  #TODO to concrete
 864         Returns the flavor identifier"""
 865
 866         # generate a new uuid put to internal dict and return it.
 867         self.logger.debug("Creating new flavor - flavor_data: {}".format(flavor_data))
 868         new_flavor=flavor_data
 869         ram = flavor_data.get(FLAVOR_RAM_KEY, 1024)
 870         cpu = flavor_data.get(FLAVOR_VCPUS_KEY, 1)
 871         disk = flavor_data.get(FLAVOR_DISK_KEY, 0)
 872
 873         if not isinstance(ram, int):
 874             raise vimconn.vimconnException("Non-integer value for ram")
 875         elif not isinstance(cpu, int):
 876             raise vimconn.vimconnException("Non-integer value for cpu")
 877         elif not isinstance(disk, int):
 878             raise vimconn.vimconnException("Non-integer value for disk")
 879
 880         extended_flv = flavor_data.get("extended")
 881         if extended_flv:
 882             numas=extended_flv.get("numas")
 883             if numas:
 884                 for numa in numas:
 885                     #overwrite ram and vcpus
 886                     if 'memory' in numa:
 887                         ram = numa['memory']*1024
 888                     if 'paired-threads' in numa:
 889                         cpu = numa['paired-threads']*2
 890                     elif 'cores' in numa:
 891                         cpu = numa['cores']
 892                     elif 'threads' in numa:
 893                         cpu = numa['threads']
 894
 895         new_flavor[FLAVOR_RAM_KEY] = ram
 896         new_flavor[FLAVOR_VCPUS_KEY] = cpu
 897         new_flavor[FLAVOR_DISK_KEY] = disk
 898         # generate a new uuid put to internal dict and return it.
 899         flavor_id = uuid.uuid4()
 900         vimconnector.flavorlist[str(flavor_id)] = new_flavor
 901         self.logger.debug("Created flavor - {} : {}".format(flavor_id, new_flavor))
 902
 903         return str(flavor_id)
 904
 905     def delete_flavor(self, flavor_id):
 906         """Deletes a tenant flavor from VIM identify by its id
 907
 908            Returns the used id or raise an exception
 909         """
 910         if flavor_id not in vimconnector.flavorlist:
 911             raise vimconn.vimconnNotFoundException("Flavor not found.")
 912
 913         vimconnector.flavorlist.pop(flavor_id, None)
 914         return flavor_id
 915
 916     def new_image(self, image_dict):
 917         """
 918         Adds a tenant image to VIM
 919         Returns:
 920             200, image-id        if the image is created
 921             <0, message          if there is an error
 922         """
 923
 924         return self.get_image_id_from_path(image_dict['location'])
 925
 926     def delete_image(self, image_id):
 927         """
 928             Deletes a tenant image from VIM
 929             Args:
 930                 image_id is ID of Image to be deleted
 931             Return:
 932                 returns the image identifier in UUID format or raises an exception on error
 933         """
 934         conn = self.connect_as_admin()
 935         if not conn:
 936             raise vimconn.vimconnConnectionException("Failed to connect vCD")
 937         # Get Catalog details
 938         url_list = [self.url, '/api/catalog/', image_id]
 939         catalog_herf = ''.join(url_list)
 940
 941         headers = {'Accept':'application/*+xml;version=' + API_VERSION,
 942                   'x-vcloud-authorization': conn._session.headers['x-vcloud-authorization']}
 943
 944         response = self.perform_request(req_type='GET',
 945                                         url=catalog_herf,
 946                                         headers=headers)
 947
 948         if response.status_code != requests.codes.ok:
 949             self.logger.debug("delete_image():GET REST API call {} failed. "\
 950                               "Return status code {}".format(catalog_herf,
 951                                                              response.status_code))
 952             raise vimconn.vimconnNotFoundException("Fail to get image {}".format(image_id))
 953
 954         lxmlroot_respond = lxmlElementTree.fromstring(response.content)
 955         namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.iteritems() if prefix}
 956         #For python3
 957         #namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.items() if prefix}
 958         namespaces["xmlns"]= "http://www.vmware.com/vcloud/v1.5"
 959
 960         catalogItems_section = lxmlroot_respond.find("xmlns:CatalogItems",namespaces)
 961         catalogItems = catalogItems_section.iterfind("xmlns:CatalogItem",namespaces)
 962         for catalogItem in catalogItems:
 963             catalogItem_href = catalogItem.attrib['href']
 964
 965             response = self.perform_request(req_type='GET',
 966                                         url=catalogItem_href,
 967                                         headers=headers)
 968
 969             if response.status_code != requests.codes.ok:
 970                 self.logger.debug("delete_image():GET REST API call {} failed. "\
 971                                   "Return status code {}".format(catalog_herf,
 972                                                                  response.status_code))
 973                 raise vimconn.vimconnNotFoundException("Fail to get catalogItem {} for catalog {}".format(
 974                                                                                     catalogItem,
 975                                                                                     image_id))
 976
 977             lxmlroot_respond = lxmlElementTree.fromstring(response.content)
 978             namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.iteritems() if prefix}
 979             #For python3
 980             #namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.items() if prefix}
 981             namespaces["xmlns"]= "http://www.vmware.com/vcloud/v1.5"
 982             catalogitem_remove_href = lxmlroot_respond.find("xmlns:Link[@rel='remove']",namespaces).attrib['href']
 983
 984             #Remove catalogItem
 985             response = self.perform_request(req_type='DELETE',
 986                                         url=catalogitem_remove_href,
 987                                         headers=headers)
 988             if response.status_code == requests.codes.no_content:
 989                 self.logger.debug("Deleted Catalog item {}".format(catalogItem))
 990             else:
 991                 raise vimconn.vimconnException("Fail to delete Catalog Item {}".format(catalogItem))
 992
 993         #Remove catalog
 994         url_list = [self.url, '/api/admin/catalog/', image_id]
 995         catalog_remove_herf = ''.join(url_list)
 996         response = self.perform_request(req_type='DELETE',
 997                                         url=catalog_remove_herf,
 998                                         headers=headers)
 999
1000         if response.status_code == requests.codes.no_content:
1001             self.logger.debug("Deleted Catalog {}".format(image_id))
1002             return image_id
1003         else:
1004             raise vimconn.vimconnException("Fail to delete Catalog {}".format(image_id))
1005
1006
1007     def catalog_exists(self, catalog_name, catalogs):
1008         """
1009
1010         :param catalog_name:
1011         :param catalogs:
1012         :return:
1013         """
1014         for catalog in catalogs:
1015             if catalog['name'] == catalog_name:
1016                 return True
1017         return False
1018
1019     def create_vimcatalog(self, vca=None, catalog_name=None):
1020         """ Create new catalog entry in vCloud director.
1021
1022             Args
1023                 vca:  vCloud director.
1024                 catalog_name catalog that client wish to create.   Note no validation done for a name.
1025                 Client must make sure that provide valid string representation.
1026
1027              Return (bool) True if catalog created.
1028
1029         """
1030         try:
1031             result = vca.create_catalog(catalog_name, catalog_name)
1032             if result is not None:
1033                 return True
1034             catalogs = vca.list_catalogs()
1035         except:
1036             return False
1037         return self.catalog_exists(catalog_name, catalogs)
1038
1039     # noinspection PyIncorrectDocstring
1040     def upload_ovf(self, vca=None, catalog_name=None, image_name=None, media_file_name=None,
1041                    description='', progress=False, chunk_bytes=128 * 1024):
1042         """
1043         Uploads a OVF file to a vCloud catalog
1044
1045         :param chunk_bytes:
1046         :param progress:
1047         :param description:
1048         :param image_name:
1049         :param vca:
1050         :param catalog_name: (str): The name of the catalog to upload the media.
1051         :param media_file_name: (str): The name of the local media file to upload.
1052         :return: (bool) True if the media file was successfully uploaded, false otherwise.
1053         """
1054         os.path.isfile(media_file_name)
1055         statinfo = os.stat(media_file_name)
1056
1057         #  find a catalog entry where we upload OVF.
1058         #  create vApp Template and check the status if vCD able to read OVF it will respond with appropirate
1059         #  status change.
1060         #  if VCD can parse OVF we upload VMDK file
1061         try:
1062             for catalog in vca.list_catalogs():
1063                 if catalog_name != catalog['name']:
1064                     continue
1065                 catalog_href = "{}/api/catalog/{}/action/upload".format(self.url, catalog['id'])
1066                 data = """
1067                 <UploadVAppTemplateParams name="{}" xmlns="http://www.vmware.com/vcloud/v1.5" xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/1"><Description>{} vApp Template</Description></UploadVAppTemplateParams>
1068                 """.format(catalog_name, description)
1069
1070                 if self.client:
1071                     headers = {'Accept':'application/*+xml;version=' + API_VERSION,
1072                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
1073                     headers['Content-Type'] = 'application/vnd.vmware.vcloud.uploadVAppTemplateParams+xml'
1074
1075                 response = self.perform_request(req_type='POST',
1076                                                 url=catalog_href,
1077                                                 headers=headers,
1078                                                 data=data)
1079
1080                 if response.status_code == requests.codes.created:
1081                     catalogItem = XmlElementTree.fromstring(response.content)
1082                     entity = [child for child in catalogItem if
1083                               child.get("type") == "application/vnd.vmware.vcloud.vAppTemplate+xml"][0]
1084                     href = entity.get('href')
1085                     template = href
1086
1087                     response = self.perform_request(req_type='GET',
1088                                                     url=href,
1089                                                     headers=headers)
1090
1091                     if response.status_code == requests.codes.ok:
1092                         headers['Content-Type'] = 'Content-Type text/xml'
1093                         result = re.search('rel="upload:default"\shref="(.*?\/descriptor.ovf)"',response.content)
1094                         if result:
1095                             transfer_href = result.group(1)
1096
1097                         response = self.perform_request(req_type='PUT',
1098                                                     url=transfer_href,
1099                                                     headers=headers,
1100                                                     data=open(media_file_name, 'rb'))
1101                         if response.status_code != requests.codes.ok:
1102                             self.logger.debug(
1103                                 "Failed create vApp template for catalog name {} and image {}".format(catalog_name,
1104                                                                                                       media_file_name))
1105                             return False
1106
1107                     # TODO fix this with aync block
1108                     time.sleep(5)
1109
1110                     self.logger.debug("vApp template for catalog name {} and image {}".format(catalog_name, media_file_name))
1111
1112                     # uploading VMDK file
1113                     # check status of OVF upload and upload remaining files.
1114                     response = self.perform_request(req_type='GET',
1115                                                     url=template,
1116                                                     headers=headers)
1117
1118                     if response.status_code == requests.codes.ok:
1119                         result = re.search('rel="upload:default"\s*href="(.*?vmdk)"',response.content)
1120                         if result:
1121                             link_href = result.group(1)
1122                         # we skip ovf since it already uploaded.
1123                         if 'ovf' in link_href:
1124                             continue
1125                         # The OVF file and VMDK must be in a same directory
1126                         head, tail = os.path.split(media_file_name)
1127                         file_vmdk = head + '/' + link_href.split("/")[-1]
1128                         if not os.path.isfile(file_vmdk):
1129                             return False
1130                         statinfo = os.stat(file_vmdk)
1131                         if statinfo.st_size == 0:
1132                             return False
1133                         hrefvmdk = link_href
1134
1135                         if progress:
1136                             widgets = ['Uploading file: ', Percentage(), ' ', Bar(), ' ', ETA(), ' ',
1137                                            FileTransferSpeed()]
1138                             progress_bar = ProgressBar(widgets=widgets, maxval=statinfo.st_size).start()
1139
1140                         bytes_transferred = 0
1141                         f = open(file_vmdk, 'rb')
1142                         while bytes_transferred < statinfo.st_size:
1143                             my_bytes = f.read(chunk_bytes)
1144                             if len(my_bytes) <= chunk_bytes:
1145                                 headers['Content-Range'] = 'bytes %s-%s/%s' % (
1146                                     bytes_transferred, len(my_bytes) - 1, statinfo.st_size)
1147                                 headers['Content-Length'] = str(len(my_bytes))
1148                                 response = requests.put(url=hrefvmdk,
1149                                                          headers=headers,
1150                                                          data=my_bytes,
1151                                                          verify=False)
1152                                 if response.status_code == requests.codes.ok:
1153                                     bytes_transferred += len(my_bytes)
1154                                     if progress:
1155                                         progress_bar.update(bytes_transferred)
1156                                 else:
1157                                     self.logger.debug(
1158                                         'file upload failed with error: [%s] %s' % (response.status_code,
1159                                                                                         response.content))
1160
1161                                     f.close()
1162                                     return False
1163                         f.close()
1164                         if progress:
1165                             progress_bar.finish()
1166                             time.sleep(10)
1167                     return True
1168                 else:
1169                     self.logger.debug("Failed retrieve vApp template for catalog name {} for OVF {}".
1170                                       format(catalog_name, media_file_name))
1171                     return False
1172         except Exception as exp:
1173             self.logger.debug("Failed while uploading OVF to catalog {} for OVF file {} with Exception {}"
1174                 .format(catalog_name,media_file_name, exp))
1175             raise vimconn.vimconnException(
1176                 "Failed while uploading OVF to catalog {} for OVF file {} with Exception {}"
1177                 .format(catalog_name,media_file_name, exp))
1178
1179         self.logger.debug("Failed retrieve catalog name {} for OVF file {}".format(catalog_name, media_file_name))
1180         return False
1181
1182     def upload_vimimage(self, vca=None, catalog_name=None, media_name=None, medial_file_name=None, progress=False):
1183         """Upload media file"""
1184         # TODO add named parameters for readability
1185
1186         return self.upload_ovf(vca=vca, catalog_name=catalog_name, image_name=media_name.split(".")[0],
1187                                media_file_name=medial_file_name, description='medial_file_name', progress=progress)
1188
1189     def validate_uuid4(self, uuid_string=None):
1190         """  Method validate correct format of UUID.
1191
1192         Return: true if string represent valid uuid
1193         """
1194         try:
1195             val = uuid.UUID(uuid_string, version=4)
1196         except ValueError:
1197             return False
1198         return True
1199
1200     def get_catalogid(self, catalog_name=None, catalogs=None):
1201         """  Method check catalog and return catalog ID in UUID format.
1202
1203         Args
1204             catalog_name: catalog name as string
1205             catalogs:  list of catalogs.
1206
1207         Return: catalogs uuid
1208         """
1209
1210         for catalog in catalogs:
1211             if catalog['name'] == catalog_name:
1212                 catalog_id = catalog['id']
1213                 return catalog_id
1214         return None
1215
1216     def get_catalogbyid(self, catalog_uuid=None, catalogs=None):
1217         """  Method check catalog and return catalog name lookup done by catalog UUID.
1218
1219         Args
1220             catalog_name: catalog name as string
1221             catalogs:  list of catalogs.
1222
1223         Return: catalogs name or None
1224         """
1225
1226         if not self.validate_uuid4(uuid_string=catalog_uuid):
1227             return None
1228
1229         for catalog in catalogs:
1230             catalog_id = catalog.get('id')
1231             if catalog_id == catalog_uuid:
1232                 return catalog.get('name')
1233         return None
1234
1235     def get_catalog_obj(self, catalog_uuid=None, catalogs=None):
1236         """  Method check catalog and return catalog name lookup done by catalog UUID.
1237
1238         Args
1239             catalog_name: catalog name as string
1240             catalogs:  list of catalogs.
1241
1242         Return: catalogs name or None
1243         """
1244
1245         if not self.validate_uuid4(uuid_string=catalog_uuid):
1246             return None
1247
1248         for catalog in catalogs:
1249             catalog_id = catalog.get('id')
1250             if catalog_id == catalog_uuid:
1251                 return catalog
1252         return None
1253
1254     def get_image_id_from_path(self, path=None, progress=False):
1255         """  Method upload OVF image to vCloud director.
1256
1257         Each OVF image represented as single catalog entry in vcloud director.
1258         The method check for existing catalog entry.  The check done by file name without file extension.
1259
1260         if given catalog name already present method will respond with existing catalog uuid otherwise
1261         it will create new catalog entry and upload OVF file to newly created catalog.
1262
1263         If method can't create catalog entry or upload a file it will throw exception.
1264
1265         Method accept boolean flag progress that will output progress bar. It useful method
1266         for standalone upload use case. In case to test large file upload.
1267
1268         Args
1269             path: - valid path to OVF file.
1270             progress - boolean progress bar show progress bar.
1271
1272         Return: if image uploaded correct method will provide image catalog UUID.
1273         """
1274
1275         if not path:
1276             raise vimconn.vimconnException("Image path can't be None.")
1277
1278         if not os.path.isfile(path):
1279             raise vimconn.vimconnException("Can't read file. File not found.")
1280
1281         if not os.access(path, os.R_OK):
1282             raise vimconn.vimconnException("Can't read file. Check file permission to read.")
1283
1284         self.logger.debug("get_image_id_from_path() client requesting {} ".format(path))
1285
1286         dirpath, filename = os.path.split(path)
1287         flname, file_extension = os.path.splitext(path)
1288         if file_extension != '.ovf':
1289             self.logger.debug("Wrong file extension {} connector support only OVF container.".format(file_extension))
1290             raise vimconn.vimconnException("Wrong container.  vCloud director supports only OVF.")
1291
1292         catalog_name = os.path.splitext(filename)[0]
1293         catalog_md5_name = hashlib.md5(path).hexdigest()
1294         self.logger.debug("File name {} Catalog Name {} file path {} "
1295                           "vdc catalog name {}".format(filename, catalog_name, path, catalog_md5_name))
1296
1297         try:
1298             org,vdc = self.get_vdc_details()
1299             catalogs = org.list_catalogs()
1300         except Exception as exp:
1301             self.logger.debug("Failed get catalogs() with Exception {} ".format(exp))
1302             raise vimconn.vimconnException("Failed get catalogs() with Exception {} ".format(exp))
1303
1304         if len(catalogs) == 0:
1305             self.logger.info("Creating a new catalog entry {} in vcloud director".format(catalog_name))
1306             result = self.create_vimcatalog(org, catalog_md5_name)
1307             if not result:
1308                 raise vimconn.vimconnException("Failed create new catalog {} ".format(catalog_md5_name))
1309
1310             result = self.upload_vimimage(vca=org, catalog_name=catalog_md5_name,
1311                                           media_name=filename, medial_file_name=path, progress=progress)
1312             if not result:
1313                 raise vimconn.vimconnException("Failed create vApp template for catalog {} ".format(catalog_name))
1314             return self.get_catalogid(catalog_name, catalogs)
1315         else:
1316             for catalog in catalogs:
1317                 # search for existing catalog if we find same name we return ID
1318                 # TODO optimize this
1319                 if catalog['name'] == catalog_md5_name:
1320                     self.logger.debug("Found existing catalog entry for {} "
1321                                       "catalog id {}".format(catalog_name,
1322                                                              self.get_catalogid(catalog_md5_name, catalogs)))
1323                     return self.get_catalogid(catalog_md5_name, catalogs)
1324
1325         # if we didn't find existing catalog we create a new one and upload image.
1326         self.logger.debug("Creating new catalog entry {} - {}".format(catalog_name, catalog_md5_name))
1327         result = self.create_vimcatalog(org, catalog_md5_name)
1328         if not result:
1329             raise vimconn.vimconnException("Failed create new catalog {} ".format(catalog_md5_name))
1330
1331         result = self.upload_vimimage(vca=org, catalog_name=catalog_md5_name,
1332                                       media_name=filename, medial_file_name=path, progress=progress)
1333         if not result:
1334             raise vimconn.vimconnException("Failed create vApp template for catalog {} ".format(catalog_md5_name))
1335
1336         return self.get_catalogid(catalog_md5_name, org.list_catalogs())
1337
1338     def get_image_list(self, filter_dict={}):
1339         '''Obtain tenant images from VIM
1340         Filter_dict can be:
1341             name: image name
1342             id: image uuid
1343             checksum: image checksum
1344             location: image path
1345         Returns the image list of dictionaries:
1346             [{<the fields at Filter_dict plus some VIM specific>}, ...]
1347             List can be empty
1348         '''
1349
1350         try:
1351             org, vdc = self.get_vdc_details()
1352             image_list = []
1353             catalogs = org.list_catalogs()
1354             if len(catalogs) == 0:
1355                 return image_list
1356             else:
1357                 for catalog in catalogs:
1358                     catalog_uuid = catalog.get('id')
1359                     name = catalog.get('name')
1360                     filtered_dict = {}
1361                     if filter_dict.get("name") and filter_dict["name"] != name:
1362                         continue
1363                     if filter_dict.get("id") and filter_dict["id"] != catalog_uuid:
1364                         continue
1365                     filtered_dict ["name"] = name
1366                     filtered_dict ["id"] = catalog_uuid
1367                     image_list.append(filtered_dict)
1368
1369                 self.logger.debug("List of already created catalog items: {}".format(image_list))
1370                 return image_list
1371         except Exception as exp:
1372             raise vimconn.vimconnException("Exception occured while retriving catalog items {}".format(exp))
1373
1374     def get_vappid(self, vdc=None, vapp_name=None):
1375         """ Method takes vdc object and vApp name and returns vapp uuid or None
1376
1377         Args:
1378             vdc: The VDC object.
1379             vapp_name: is application vappp name identifier
1380
1381         Returns:
1382                 The return vApp name otherwise None
1383         """
1384         if vdc is None or vapp_name is None:
1385             return None
1386         # UUID has following format https://host/api/vApp/vapp-30da58a3-e7c7-4d09-8f68-d4c8201169cf
1387         try:
1388             refs = filter(lambda ref: ref.name == vapp_name and ref.type_ == 'application/vnd.vmware.vcloud.vApp+xml',
1389                           vdc.ResourceEntities.ResourceEntity)
1390             #For python3
1391             #refs = [ref for ref in vdc.ResourceEntities.ResourceEntity\
1392             #         if ref.name == vapp_name and ref.type_ == 'application/vnd.vmware.vcloud.vApp+xml']
1393             if len(refs) == 1:
1394                 return refs[0].href.split("vapp")[1][1:]
1395         except Exception as e:
1396             self.logger.exception(e)
1397             return False
1398         return None
1399
1400     def check_vapp(self, vdc=None, vapp_uuid=None):
1401         """ Method Method returns True or False if vapp deployed in vCloud director
1402
1403             Args:
1404                 vca: Connector to VCA
1405                 vdc: The VDC object.
1406                 vappid: vappid is application identifier
1407
1408             Returns:
1409                 The return True if vApp deployed
1410                 :param vdc:
1411                 :param vapp_uuid:
1412         """
1413         try:
1414             refs = filter(lambda ref:
1415                           ref.type_ == 'application/vnd.vmware.vcloud.vApp+xml',
1416                           vdc.ResourceEntities.ResourceEntity)
1417             #For python3
1418             #refs = [ref for ref in vdc.ResourceEntities.ResourceEntity\
1419             #         if ref.type_ == 'application/vnd.vmware.vcloud.vApp+xml']
1420             for ref in refs:
1421                 vappid = ref.href.split("vapp")[1][1:]
1422                 # find vapp with respected vapp uuid
1423                 if vappid == vapp_uuid:
1424                     return True
1425         except Exception as e:
1426             self.logger.exception(e)
1427             return False
1428         return False
1429
1430     def get_namebyvappid(self, vapp_uuid=None):
1431         """Method returns vApp name from vCD and lookup done by vapp_id.
1432
1433         Args:
1434             vapp_uuid: vappid is application identifier
1435
1436         Returns:
1437             The return vApp name otherwise None
1438         """
1439         try:
1440             if self.client and vapp_uuid:
1441                 vapp_call = "{}/api/vApp/vapp-{}".format(self.url, vapp_uuid)
1442                 headers = {'Accept':'application/*+xml;version=' + API_VERSION,
1443                      'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
1444
1445                 response = self.perform_request(req_type='GET',
1446                                                 url=vapp_call,
1447                                                 headers=headers)
1448                 #Retry login if session expired & retry sending request
1449                 if response.status_code == 403:
1450                     response = self.retry_rest('GET', vapp_call)
1451
1452                 tree = XmlElementTree.fromstring(response.content)
1453                 return tree.attrib['name']
1454         except Exception as e:
1455             self.logger.exception(e)
1456             return None
1457         return None
1458
1459     def new_vminstance(self, name=None, description="", start=False, image_id=None, flavor_id=None, net_list=[],
1460                        cloud_config=None, disk_list=None, availability_zone_index=None, availability_zone_list=None):
1461         """Adds a VM instance to VIM
1462         Params:
1463             'start': (boolean) indicates if VM must start or created in pause mode.
1464             'image_id','flavor_id': image and flavor VIM id to use for the VM
1465             'net_list': list of interfaces, each one is a dictionary with:
1466                 'name': (optional) name for the interface.
1467                 'net_id': VIM network id where this interface must be connect to. Mandatory for type==virtual
1468                 'vpci': (optional) virtual vPCI address to assign at the VM. Can be ignored depending on VIM capabilities
1469                 'model': (optional and only have sense for type==virtual) interface model: virtio, e1000, ...
1470                 'mac_address': (optional) mac address to assign to this interface
1471                 #TODO: CHECK if an optional 'vlan' parameter is needed for VIMs when type if VF and net_id is not provided,
1472                     the VLAN tag to be used. In case net_id is provided, the internal network vlan is used for tagging VF
1473                 'type': (mandatory) can be one of:
1474                     'virtual', in this case always connected to a network of type 'net_type=bridge'
1475                      'PCI-PASSTHROUGH' or 'PF' (passthrough): depending on VIM capabilities it can be connected to a data/ptp network ot it
1476                            can created unconnected
1477                      'SR-IOV' or 'VF' (SRIOV with VLAN tag): same as PF for network connectivity.
1478                      'VFnotShared'(SRIOV without VLAN tag) same as PF for network connectivity. VF where no other VFs
1479                             are allocated on the same physical NIC
1480                 'bw': (optional) only for PF/VF/VFnotShared. Minimal Bandwidth required for the interface in GBPS
1481                 'port_security': (optional) If False it must avoid any traffic filtering at this interface. If missing
1482                                 or True, it must apply the default VIM behaviour
1483                 After execution the method will add the key:
1484                 'vim_id': must be filled/added by this method with the VIM identifier generated by the VIM for this
1485                         interface. 'net_list' is modified
1486             'cloud_config': (optional) dictionary with:
1487                 'key-pairs': (optional) list of strings with the public key to be inserted to the default user
1488                 'users': (optional) list of users to be inserted, each item is a dict with:
1489                     'name': (mandatory) user name,
1490                     'key-pairs': (optional) list of strings with the public key to be inserted to the user
1491                 'user-data': (optional) can be a string with the text script to be passed directly to cloud-init,
1492                     or a list of strings, each one contains a script to be passed, usually with a MIMEmultipart file
1493                 'config-files': (optional). List of files to be transferred. Each item is a dict with:
1494                     'dest': (mandatory) string with the destination absolute path
1495                     'encoding': (optional, by default text). Can be one of:
1496                         'b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64'
1497                     'content' (mandatory): string with the content of the file
1498                     'permissions': (optional) string with file permissions, typically octal notation '0644'
1499                     'owner': (optional) file owner, string with the format 'owner:group'
1500                 'boot-data-drive': boolean to indicate if user-data must be passed using a boot drive (hard disk)
1501             'disk_list': (optional) list with additional disks to the VM. Each item is a dict with:
1502                 'image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted
1503                 'size': (mandatory) string with the size of the disk in GB
1504             availability_zone_index: Index of availability_zone_list to use for this this VM. None if not AV required
1505             availability_zone_list: list of availability zones given by user in the VNFD descriptor.  Ignore if
1506                 availability_zone_index is None
1507         Returns a tuple with the instance identifier and created_items or raises an exception on error
1508             created_items can be None or a dictionary where this method can include key-values that will be passed to
1509             the method delete_vminstance and action_vminstance. Can be used to store created ports, volumes, etc.
1510             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
1511             as not present.
1512         """
1513         self.logger.info("Creating new instance for entry {}".format(name))
1514         self.logger.debug("desc {} boot {} image_id: {} flavor_id: {} net_list: {} cloud_config {} disk_list {} "\
1515                           "availability_zone_index {} availability_zone_list {}"\
1516                           .format(description, start, image_id, flavor_id, net_list, cloud_config, disk_list,\
1517                                   availability_zone_index, availability_zone_list))
1518
1519         #new vm name = vmname + tenant_id + uuid
1520         new_vm_name = [name, '-', str(uuid.uuid4())]
1521         vmname_andid = ''.join(new_vm_name)
1522
1523         for net in net_list:
1524             if net['type'] == "SR-IOV" or net['type'] == "PCI-PASSTHROUGH":
1525                 raise vimconn.vimconnNotSupportedException(
1526                       "Current vCD version does not support type : {}".format(net['type']))
1527
1528         if len(net_list) > 10:
1529             raise vimconn.vimconnNotSupportedException(
1530                       "The VM hardware versions 7 and above support upto 10 NICs only")
1531
1532         # if vm already deployed we return existing uuid
1533         # we check for presence of VDC, Catalog entry and Flavor.
1534         org, vdc = self.get_vdc_details()
1535         if vdc is None:
1536             raise vimconn.vimconnNotFoundException(
1537                 "new_vminstance(): Failed create vApp {}: (Failed retrieve VDC information)".format(name))
1538         catalogs = org.list_catalogs()
1539         if catalogs is None:
1540             #Retry once, if failed by refreshing token
1541             self.get_token()
1542             org = Org(self.client, resource=self.client.get_org())
1543             catalogs = org.list_catalogs()
1544         if catalogs is None:
1545             raise vimconn.vimconnNotFoundException(
1546                 "new_vminstance(): Failed create vApp {}: (Failed retrieve catalogs list)".format(name))
1547
1548         catalog_hash_name = self.get_catalogbyid(catalog_uuid=image_id, catalogs=catalogs)
1549         if catalog_hash_name:
1550             self.logger.info("Found catalog entry {} for image id {}".format(catalog_hash_name, image_id))
1551         else:
1552             raise vimconn.vimconnNotFoundException("new_vminstance(): Failed create vApp {}: "
1553                                                    "(Failed retrieve catalog information {})".format(name, image_id))
1554
1555
1556         # Set vCPU and Memory based on flavor.
1557         vm_cpus = None
1558         vm_memory = None
1559         vm_disk = None
1560         numas = None
1561
1562         if flavor_id is not None:
1563             if flavor_id not in vimconnector.flavorlist:
1564                 raise vimconn.vimconnNotFoundException("new_vminstance(): Failed create vApp {}: "
1565                                                        "Failed retrieve flavor information "
1566                                                        "flavor id {}".format(name, flavor_id))
1567             else:
1568                 try:
1569                     flavor = vimconnector.flavorlist[flavor_id]
1570                     vm_cpus = flavor[FLAVOR_VCPUS_KEY]
1571                     vm_memory = flavor[FLAVOR_RAM_KEY]
1572                     vm_disk = flavor[FLAVOR_DISK_KEY]
1573                     extended = flavor.get("extended", None)
1574                     if extended:
1575                         numas=extended.get("numas", None)
1576
1577                 except Exception as exp:
1578                     raise vimconn.vimconnException("Corrupted flavor. {}.Exception: {}".format(flavor_id, exp))
1579
1580         # image upload creates template name as catalog name space Template.
1581         templateName = self.get_catalogbyid(catalog_uuid=image_id, catalogs=catalogs)
1582         power_on = 'false'
1583         if start:
1584             power_on = 'true'
1585
1586         # client must provide at least one entry in net_list if not we report error
1587         #If net type is mgmt, then configure it as primary net & use its NIC index as primary NIC
1588         #If no mgmt, then the 1st NN in netlist is considered as primary net.
1589         primary_net = None
1590         primary_netname = None
1591         primary_net_href = None
1592         network_mode = 'bridged'
1593         if net_list is not None and len(net_list) > 0:
1594             for net in net_list:
1595                 if 'use' in net and net['use'] == 'mgmt' and not primary_net:
1596                     primary_net = net
1597             if primary_net is None:
1598                 primary_net = net_list[0]
1599
1600             try:
1601                 primary_net_id = primary_net['net_id']
1602                 url_list = [self.url, '/api/network/', primary_net_id]
1603                 primary_net_href = ''.join(url_list)
1604                 network_dict = self.get_vcd_network(network_uuid=primary_net_id)
1605                 if 'name' in network_dict:
1606                     primary_netname = network_dict['name']
1607
1608             except KeyError:
1609                 raise vimconn.vimconnException("Corrupted flavor. {}".format(primary_net))
1610         else:
1611             raise vimconn.vimconnUnexpectedResponse("new_vminstance(): Failed network list is empty.".format(name))
1612
1613         # use: 'data', 'bridge', 'mgmt'
1614         # create vApp.  Set vcpu and ram based on flavor id.
1615         try:
1616             vdc_obj = VDC(self.client, resource=org.get_vdc(self.tenant_name))
1617             if not vdc_obj:
1618                 raise vimconn.vimconnNotFoundException("new_vminstance(): Failed to get VDC object")
1619
1620             for retry in (1,2):
1621                 items = org.get_catalog_item(catalog_hash_name, catalog_hash_name)
1622                 catalog_items = [items.attrib]
1623
1624                 if len(catalog_items) == 1:
1625                     if self.client:
1626                         headers = {'Accept':'application/*+xml;version=' + API_VERSION,
1627                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
1628
1629                     response = self.perform_request(req_type='GET',
1630                                                 url=catalog_items[0].get('href'),
1631                                                 headers=headers)
1632                     catalogItem = XmlElementTree.fromstring(response.content)
1633                     entity = [child for child in catalogItem if child.get("type") == "application/vnd.vmware.vcloud.vAppTemplate+xml"][0]
1634                     vapp_tempalte_href = entity.get("href")
1635
1636                 response = self.perform_request(req_type='GET',
1637                                                     url=vapp_tempalte_href,
1638                                                     headers=headers)
1639                 if response.status_code != requests.codes.ok:
1640                     self.logger.debug("REST API call {} failed. Return status code {}".format(vapp_tempalte_href,
1641                                                                                            response.status_code))
1642                 else:
1643                     result = (response.content).replace("\n"," ")
1644
1645                 src = re.search('<Vm goldMaster="false"\sstatus="\d+"\sname="(.*?)"\s'
1646                                                'id="(\w+:\w+:vm:.*?)"\shref="(.*?)"\s'
1647                               'type="application/vnd\.vmware\.vcloud\.vm\+xml',result)
1648                 if src:
1649                     vm_name = src.group(1)
1650                     vm_id = src.group(2)
1651                     vm_href = src.group(3)
1652
1653                 cpus = re.search('<rasd:Description>Number of Virtual CPUs</.*?>(\d+)</rasd:VirtualQuantity>',result).group(1)
1654                 memory_mb = re.search('<rasd:Description>Memory Size</.*?>(\d+)</rasd:VirtualQuantity>',result).group(1)
1655                 cores = re.search('<vmw:CoresPerSocket ovf:required.*?>(\d+)</vmw:CoresPerSocket>',result).group(1)
1656
1657                 headers['Content-Type'] = 'application/vnd.vmware.vcloud.instantiateVAppTemplateParams+xml'
1658                 vdc_id = vdc.get('id').split(':')[-1]
1659                 instantiate_vapp_href = "{}/api/vdc/{}/action/instantiateVAppTemplate".format(self.url,
1660                                                                                                 vdc_id)
1661                 data = """<?xml version="1.0" encoding="UTF-8"?>
1662                 <InstantiateVAppTemplateParams
1663                 xmlns="http://www.vmware.com/vcloud/v1.5"
1664                 name="{}"
1665                 deploy="false"
1666                 powerOn="false"
1667                 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
1668                 xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/1">
1669                 <Description>Vapp instantiation</Description>
1670                 <InstantiationParams>
1671                      <NetworkConfigSection>
1672                          <ovf:Info>Configuration parameters for logical networks</ovf:Info>
1673                          <NetworkConfig networkName="{}">
1674                              <Configuration>
1675                                  <ParentNetwork href="{}" />
1676                                  <FenceMode>bridged</FenceMode>
1677                              </Configuration>
1678                          </NetworkConfig>
1679                      </NetworkConfigSection>
1680                 <LeaseSettingsSection
1681                 type="application/vnd.vmware.vcloud.leaseSettingsSection+xml">
1682                 <ovf:Info>Lease Settings</ovf:Info>
1683                 <StorageLeaseInSeconds>172800</StorageLeaseInSeconds>
1684                 <StorageLeaseExpiration>2014-04-25T08:08:16.438-07:00</StorageLeaseExpiration>
1685                 </LeaseSettingsSection>
1686                 </InstantiationParams>
1687                 <Source href="{}"/>
1688                 <SourcedItem>
1689                 <Source href="{}" id="{}" name="{}"
1690                 type="application/vnd.vmware.vcloud.vm+xml"/>
1691                 <VmGeneralParams>
1692                     <NeedsCustomization>false</NeedsCustomization>
1693                 </VmGeneralParams>
1694                 <InstantiationParams>
1695                       <NetworkConnectionSection>
1696                       <ovf:Info>Specifies the available VM network connections</ovf:Info>
1697                       <NetworkConnection network="{}">
1698                       <NetworkConnectionIndex>0</NetworkConnectionIndex>
1699                       <IsConnected>true</IsConnected>
1700                       <IpAddressAllocationMode>DHCP</IpAddressAllocationMode>
1701                       </NetworkConnection>
1702                       </NetworkConnectionSection><ovf:VirtualHardwareSection>
1703                       <ovf:Info>Virtual hardware requirements</ovf:Info>
1704                       <ovf:Item xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData"
1705                       xmlns:vmw="http://www.vmware.com/schema/ovf">
1706                       <rasd:AllocationUnits>hertz * 10^6</rasd:AllocationUnits>
1707                       <rasd:Description>Number of Virtual CPUs</rasd:Description>
1708                       <rasd:ElementName xmlns:py="http://codespeak.net/lxml/objectify/pytype" py:pytype="str">{cpu} virtual CPU(s)</rasd:ElementName>
1709                       <rasd:InstanceID>4</rasd:InstanceID>
1710                       <rasd:Reservation>0</rasd:Reservation>
1711                       <rasd:ResourceType>3</rasd:ResourceType>
1712                       <rasd:VirtualQuantity xmlns:py="http://codespeak.net/lxml/objectify/pytype" py:pytype="int">{cpu}</rasd:VirtualQuantity>
1713                       <rasd:Weight>0</rasd:Weight>
1714                       <vmw:CoresPerSocket ovf:required="false">{core}</vmw:CoresPerSocket>
1715                       </ovf:Item><ovf:Item xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData">
1716                       <rasd:AllocationUnits>byte * 2^20</rasd:AllocationUnits>
1717                       <rasd:Description>Memory Size</rasd:Description>
1718                       <rasd:ElementName xmlns:py="http://codespeak.net/lxml/objectify/pytype" py:pytype="str">{memory} MB of memory</rasd:ElementName>
1719                       <rasd:InstanceID>5</rasd:InstanceID>
1720                       <rasd:Reservation>0</rasd:Reservation>
1721                       <rasd:ResourceType>4</rasd:ResourceType>
1722                       <rasd:VirtualQuantity xmlns:py="http://codespeak.net/lxml/objectify/pytype" py:pytype="int">{memory}</rasd:VirtualQuantity>
1723                       <rasd:Weight>0</rasd:Weight>
1724                       </ovf:Item>
1725                 </ovf:VirtualHardwareSection>
1726                 </InstantiationParams>
1727                 </SourcedItem>
1728                 <AllEULAsAccepted>false</AllEULAsAccepted>
1729                 </InstantiateVAppTemplateParams>""".format(vmname_andid,
1730                                                         primary_netname,
1731                                                         primary_net_href,
1732                                                      vapp_tempalte_href,
1733                                                                 vm_href,
1734                                                                   vm_id,
1735                                                                 vm_name,
1736                                                         primary_netname,
1737                                                                cpu=cpus,
1738                                                              core=cores,
1739                                                        memory=memory_mb)
1740
1741                 response = self.perform_request(req_type='POST',
1742                                                 url=instantiate_vapp_href,
1743                                                 headers=headers,
1744                                                 data=data)
1745
1746                 if response.status_code != 201:
1747                     self.logger.error("REST call {} failed reason : {}"\
1748                          "status code : {}".format(instantiate_vapp_href,
1749                                                         response.content,
1750                                                    response.status_code))
1751                     raise vimconn.vimconnException("new_vminstance(): Failed to create"\
1752                                                         "vAapp {}".format(vmname_andid))
1753                 else:
1754                     vapptask = self.get_task_from_response(response.content)
1755
1756                 if vapptask is None and retry==1:
1757                     self.get_token() # Retry getting token
1758                     continue
1759                 else:
1760                     break
1761
1762             if vapptask is None or vapptask is False:
1763                 raise vimconn.vimconnUnexpectedResponse(
1764                     "new_vminstance(): failed to create vApp {}".format(vmname_andid))
1765
1766             # wait for task to complete
1767             result = self.client.get_task_monitor().wait_for_success(task=vapptask)
1768
1769             if result.get('status') == 'success':
1770                 self.logger.debug("new_vminstance(): Sucessfully created Vapp {}".format(vmname_andid))
1771             else:
1772                 raise vimconn.vimconnUnexpectedResponse(
1773                     "new_vminstance(): failed to create vApp {}".format(vmname_andid))
1774
1775         except Exception as exp:
1776             raise vimconn.vimconnUnexpectedResponse(
1777                 "new_vminstance(): failed to create vApp {} with Exception:{}".format(vmname_andid, exp))
1778
1779         # we should have now vapp in undeployed state.
1780         try:
1781             vdc_obj = VDC(self.client, href=vdc.get('href'))
1782             vapp_resource = vdc_obj.get_vapp(vmname_andid)
1783             vapp_uuid = vapp_resource.get('id').split(':')[-1]
1784             vapp = VApp(self.client, resource=vapp_resource)
1785
1786         except Exception as exp:
1787             raise vimconn.vimconnUnexpectedResponse(
1788                     "new_vminstance(): Failed to retrieve vApp {} after creation: Exception:{}"
1789                     .format(vmname_andid, exp))
1790
1791         if vapp_uuid is None:
1792             raise vimconn.vimconnUnexpectedResponse(
1793                 "new_vminstance(): Failed to retrieve vApp {} after creation".format(
1794                                                                             vmname_andid))
1795
1796         #Add PCI passthrough/SRIOV configrations
1797         vm_obj = None
1798         pci_devices_info = []
1799         sriov_net_info = []
1800         reserve_memory = False
1801
1802         for net in net_list:
1803             if net["type"] == "PF" or net["type"] == "PCI-PASSTHROUGH":
1804                 pci_devices_info.append(net)
1805             elif (net["type"] == "VF" or net["type"] == "SR-IOV" or net["type"] == "VFnotShared") and 'net_id'in net:
1806                 sriov_net_info.append(net)
1807
1808         #Add PCI
1809         if len(pci_devices_info) > 0:
1810             self.logger.info("Need to add PCI devices {} into VM {}".format(pci_devices_info,
1811                                                                         vmname_andid ))
1812             PCI_devices_status, vm_obj, vcenter_conect = self.add_pci_devices(vapp_uuid,
1813                                                                             pci_devices_info,
1814                                                                             vmname_andid)
1815             if PCI_devices_status:
1816                 self.logger.info("Added PCI devives {} to VM {}".format(
1817                                                             pci_devices_info,
1818                                                             vmname_andid)
1819                                  )
1820                 reserve_memory = True
1821             else:
1822                 self.logger.info("Fail to add PCI devives {} to VM {}".format(
1823                                                             pci_devices_info,
1824                                                             vmname_andid)
1825                                  )
1826
1827         # Modify vm disk
1828         if vm_disk:
1829             #Assuming there is only one disk in ovf and fast provisioning in organization vDC is disabled
1830             result = self.modify_vm_disk(vapp_uuid, vm_disk)
1831             if result :
1832                 self.logger.debug("Modified Disk size of VM {} ".format(vmname_andid))
1833
1834         #Add new or existing disks to vApp
1835         if disk_list:
1836             added_existing_disk = False
1837             for disk in disk_list:
1838                 if 'device_type' in disk and disk['device_type'] == 'cdrom':
1839                     image_id = disk['image_id']
1840                     # Adding CD-ROM to VM
1841                     # will revisit code once specification ready to support this feature
1842                     self.insert_media_to_vm(vapp, image_id)
1843                 elif "image_id" in disk and disk["image_id"] is not None:
1844                     self.logger.debug("Adding existing disk from image {} to vm {} ".format(
1845                                                                     disk["image_id"] , vapp_uuid))
1846                     self.add_existing_disk(catalogs=catalogs,
1847                                            image_id=disk["image_id"],
1848                                            size = disk["size"],
1849                                            template_name=templateName,
1850                                            vapp_uuid=vapp_uuid
1851                                            )
1852                     added_existing_disk = True
1853                 else:
1854                     #Wait till added existing disk gets reflected into vCD database/API
1855                     if added_existing_disk:
1856                         time.sleep(5)
1857                         added_existing_disk = False
1858                     self.add_new_disk(vapp_uuid, disk['size'])
1859
1860         if numas:
1861             # Assigning numa affinity setting
1862             for numa in numas:
1863                 if 'paired-threads-id' in numa:
1864                     paired_threads_id = numa['paired-threads-id']
1865                     self.set_numa_affinity(vapp_uuid, paired_threads_id)
1866
1867         # add NICs & connect to networks in netlist
1868         try:
1869             self.logger.info("Request to connect VM to a network: {}".format(net_list))
1870             primary_nic_index = 0
1871             nicIndex = 0
1872             for net in net_list:
1873                 # openmano uses network id in UUID format.
1874                 # vCloud Director need a name so we do reverse operation from provided UUID we lookup a name
1875                 # [{'use': 'bridge', 'net_id': '527d4bf7-566a-41e7-a9e7-ca3cdd9cef4f', 'type': 'virtual',
1876                 #   'vpci': '0000:00:11.0', 'name': 'eth0'}]
1877
1878                 if 'net_id' not in net:
1879                     continue
1880
1881                 #Using net_id as a vim_id i.e. vim interface id, as do not have saperate vim interface id
1882                 #Same will be returned in refresh_vms_status() as vim_interface_id
1883                 net['vim_id'] = net['net_id']  # Provide the same VIM identifier as the VIM network
1884
1885                 interface_net_id = net['net_id']
1886                 interface_net_name = self.get_network_name_by_id(network_uuid=interface_net_id)
1887                 interface_network_mode = net['use']
1888
1889                 if interface_net_name == primary_netname:
1890                     nicIndex += 1
1891                     continue
1892
1893                 if interface_network_mode == 'mgmt':
1894                     primary_nic_index = nicIndex
1895
1896                 """- POOL (A static IP address is allocated automatically from a pool of addresses.)
1897                                   - DHCP (The IP address is obtained from a DHCP service.)
1898                                   - MANUAL (The IP address is assigned manually in the IpAddress element.)
1899                                   - NONE (No IP addressing mode specified.)"""
1900
1901                 if primary_netname is not None:
1902                     nets = filter(lambda n: n.get('name') == interface_net_name, self.get_network_list())
1903                     #For python3
1904                     #nets = [n for n in self.get_network_list() if n.get('name') == interface_net_name]
1905                     if len(nets) == 1:
1906                         self.logger.info("new_vminstance(): Found requested network: {}".format(nets[0].get('name')))
1907
1908                         vdc_obj = VDC(self.client, href=vdc.get('href'))
1909                         vapp_resource = vdc_obj.get_vapp(vmname_andid)
1910                         vapp = VApp(self.client, resource=vapp_resource)
1911                         # connect network to VM - with all DHCP by default
1912                         task = vapp.connect_org_vdc_network(nets[0].get('name'))
1913
1914                         self.client.get_task_monitor().wait_for_success(task=task)
1915
1916                         type_list = ('PF', 'PCI-PASSTHROUGH', 'VF', 'SR-IOV', 'VFnotShared')
1917                         if 'type' in net and net['type'] not in type_list:
1918                             # fetching nic type from vnf
1919                             if 'model' in net:
1920                                 if net['model'] is not None and net['model'].lower() == 'virtio':
1921                                     nic_type = 'VMXNET3'
1922                                 else:
1923                                     nic_type = net['model']
1924
1925                                 self.logger.info("new_vminstance(): adding network adapter "\
1926                                                           "to a network {}".format(nets[0].get('name')))
1927                                 self.add_network_adapter_to_vms(vapp, nets[0].get('name'),
1928                                                                 primary_nic_index,
1929                                                                 nicIndex,
1930                                                                 net,
1931                                                                 nic_type=nic_type)
1932                             else:
1933                                 self.logger.info("new_vminstance(): adding network adapter "\
1934                                                          "to a network {}".format(nets[0].get('name')))
1935                                 self.add_network_adapter_to_vms(vapp, nets[0].get('name'),
1936                                                                 primary_nic_index,
1937                                                                 nicIndex,
1938                                                                 net)
1939                 nicIndex += 1
1940
1941             # cloud-init for ssh-key injection
1942             if cloud_config:
1943                 self.cloud_init(vapp,cloud_config)
1944
1945         # ############# Stub code for SRIOV #################
1946         #Add SRIOV
1947 #         if len(sriov_net_info) > 0:
1948 #             self.logger.info("Need to add SRIOV adapters {} into VM {}".format(sriov_net_info,
1949 #                                                                         vmname_andid ))
1950 #             sriov_status, vm_obj, vcenter_conect = self.add_sriov(vapp_uuid,
1951 #                                                                   sriov_net_info,
1952 #                                                                   vmname_andid)
1953 #             if sriov_status:
1954 #                 self.logger.info("Added SRIOV {} to VM {}".format(
1955 #                                                             sriov_net_info,
1956 #                                                             vmname_andid)
1957 #                                  )
1958 #                 reserve_memory = True
1959 #             else:
1960 #                 self.logger.info("Fail to add SRIOV {} to VM {}".format(
1961 #                                                             sriov_net_info,
1962 #                                                             vmname_andid)
1963 #                                  )
1964
1965             # If VM has PCI devices or SRIOV reserve memory for VM
1966             if reserve_memory:
1967                 memReserve = vm_obj.config.hardware.memoryMB
1968                 spec = vim.vm.ConfigSpec()
1969                 spec.memoryAllocation = vim.ResourceAllocationInfo(reservation=memReserve)
1970                 task = vm_obj.ReconfigVM_Task(spec=spec)
1971                 if task:
1972                     result = self.wait_for_vcenter_task(task, vcenter_conect)
1973                     self.logger.info("Reserved memory {} MB for "
1974                                      "VM VM status: {}".format(str(memReserve), result))
1975                 else:
1976                     self.logger.info("Fail to reserved memory {} to VM {}".format(
1977                                                                 str(memReserve), str(vm_obj)))
1978
1979             self.logger.debug("new_vminstance(): starting power on vApp {} ".format(vmname_andid))
1980
1981             vapp_id = vapp_resource.get('id').split(':')[-1]
1982             poweron_task = self.power_on_vapp(vapp_id, vmname_andid)
1983             result = self.client.get_task_monitor().wait_for_success(task=poweron_task)
1984             if result.get('status') == 'success':
1985                 self.logger.info("new_vminstance(): Successfully power on "\
1986                                              "vApp {}".format(vmname_andid))
1987             else:
1988                 self.logger.error("new_vminstance(): failed to power on vApp "\
1989                                                      "{}".format(vmname_andid))
1990
1991         except Exception as exp :
1992             # it might be a case if specific mandatory entry in dict is empty or some other pyVcloud exception
1993             self.logger.error("new_vminstance(): Failed create new vm instance {} with exception {}"
1994                               .format(name, exp))
1995             raise vimconn.vimconnException("new_vminstance(): Failed create new vm instance {} with exception {}"
1996                                            .format(name, exp))
1997
1998         # check if vApp deployed and if that the case return vApp UUID otherwise -1
1999         wait_time = 0
2000         vapp_uuid = None
2001         while wait_time <= MAX_WAIT_TIME:
2002             try:
2003                 vapp_resource = vdc_obj.get_vapp(vmname_andid)
2004                 vapp = VApp(self.client, resource=vapp_resource)
2005             except Exception as exp:
2006                 raise vimconn.vimconnUnexpectedResponse(
2007                         "new_vminstance(): Failed to retrieve vApp {} after creation: Exception:{}"
2008                         .format(vmname_andid, exp))
2009
2010             #if vapp and vapp.me.deployed:
2011             if vapp and vapp_resource.get('deployed') == 'true':
2012                 vapp_uuid = vapp_resource.get('id').split(':')[-1]
2013                 break
2014             else:
2015                 self.logger.debug("new_vminstance(): Wait for vApp {} to deploy".format(name))
2016                 time.sleep(INTERVAL_TIME)
2017
2018             wait_time +=INTERVAL_TIME
2019
2020         #SET Affinity Rule for VM
2021         #Pre-requisites: User has created Hosh Groups in vCenter with respective Hosts to be used
2022         #While creating VIM account user has to pass the Host Group names in availability_zone list
2023         #"availability_zone" is a  part of VIM "config" parameters
2024         #For example, in VIM config: "availability_zone":["HG_170","HG_174","HG_175"]
2025         #Host groups are referred as availability zones
2026         #With following procedure, deployed VM will be added into a VM group.
2027         #Then A VM to Host Affinity rule will be created using the VM group & Host group.
2028         if(availability_zone_list):
2029             self.logger.debug("Existing Host Groups in VIM {}".format(self.config.get('availability_zone')))
2030             #Admin access required for creating Affinity rules
2031             client = self.connect_as_admin()
2032             if not client:
2033                 raise vimconn.vimconnConnectionException("Failed to connect vCD as admin")
2034             else:
2035                 self.client = client
2036             if self.client:
2037                 headers = {'Accept':'application/*+xml;version=27.0',
2038                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
2039             #Step1: Get provider vdc details from organization
2040             pvdc_href = self.get_pvdc_for_org(self.tenant_name, headers)
2041             if pvdc_href is not None:
2042             #Step2: Found required pvdc, now get resource pool information
2043                 respool_href = self.get_resource_pool_details(pvdc_href, headers)
2044                 if respool_href is None:
2045                     #Raise error if respool_href not found
2046                     msg = "new_vminstance():Error in finding resource pool details in pvdc {}"\
2047                            .format(pvdc_href)
2048                     self.log_message(msg)
2049
2050             #Step3: Verify requested availability zone(hostGroup) is present in vCD
2051             # get availability Zone
2052             vm_az = self.get_vm_availability_zone(availability_zone_index, availability_zone_list)
2053             # check if provided av zone(hostGroup) is present in vCD VIM
2054             status = self.check_availibility_zone(vm_az, respool_href, headers)
2055             if status is False:
2056                 msg = "new_vminstance(): Error in finding availability zone(Host Group): {} in "\
2057                        "resource pool {} status: {}".format(vm_az,respool_href,status)
2058                 self.log_message(msg)
2059             else:
2060                 self.logger.debug ("new_vminstance(): Availability zone {} found in VIM".format(vm_az))
2061
2062             #Step4: Find VM group references to create vm group
2063             vmgrp_href = self.find_vmgroup_reference(respool_href, headers)
2064             if vmgrp_href == None:
2065                 msg = "new_vminstance(): No reference to VmGroup found in resource pool"
2066                 self.log_message(msg)
2067
2068             #Step5: Create a VmGroup with name az_VmGroup
2069             vmgrp_name = vm_az + "_" + name #Formed VM Group name = Host Group name + VM name
2070             status = self.create_vmgroup(vmgrp_name, vmgrp_href, headers)
2071             if status is not True:
2072                 msg = "new_vminstance(): Error in creating VM group {}".format(vmgrp_name)
2073                 self.log_message(msg)
2074
2075             #VM Group url to add vms to vm group
2076             vmgrpname_url = self.url + "/api/admin/extension/vmGroup/name/"+ vmgrp_name
2077
2078             #Step6: Add VM to VM Group
2079             #Find VM uuid from vapp_uuid
2080             vm_details = self.get_vapp_details_rest(vapp_uuid)
2081             vm_uuid = vm_details['vmuuid']
2082
2083             status = self.add_vm_to_vmgroup(vm_uuid, vmgrpname_url, vmgrp_name, headers)
2084             if status is not True:
2085                 msg = "new_vminstance(): Error in adding VM to VM group {}".format(vmgrp_name)
2086                 self.log_message(msg)
2087
2088             #Step7: Create VM to Host affinity rule
2089             addrule_href = self.get_add_rule_reference (respool_href, headers)
2090             if addrule_href is None:
2091                 msg = "new_vminstance(): Error in finding href to add rule in resource pool: {}"\
2092                       .format(respool_href)
2093                 self.log_message(msg)
2094
2095             status = self.create_vm_to_host_affinity_rule(addrule_href, vmgrp_name, vm_az, "Affinity",  headers)
2096             if status is False:
2097                 msg = "new_vminstance(): Error in creating affinity rule for VM {} in Host group {}"\
2098                       .format(name, vm_az)
2099                 self.log_message(msg)
2100             else:
2101                 self.logger.debug("new_vminstance(): Affinity rule created successfully. Added {} in Host group {}"\
2102                                     .format(name, vm_az))
2103             #Reset token to a normal user to perform other operations
2104             self.get_token()
2105
2106         if vapp_uuid is not None:
2107             return vapp_uuid, None
2108         else:
2109             raise vimconn.vimconnUnexpectedResponse("new_vminstance(): Failed create new vm instance {}".format(name))
2110
2111
2112     def get_vcd_availibility_zones(self,respool_href, headers):
2113         """ Method to find presence of av zone is VIM resource pool
2114
2115             Args:
2116                 respool_href - resource pool href
2117                 headers - header information
2118
2119             Returns:
2120                vcd_az - list of azone present in vCD
2121         """
2122         vcd_az = []
2123         url=respool_href
2124         resp = self.perform_request(req_type='GET',url=respool_href, headers=headers)
2125
2126         if resp.status_code != requests.codes.ok:
2127             self.logger.debug ("REST API call {} failed. Return status code {}".format(url, resp.status_code))
2128         else:
2129         #Get the href to hostGroups and find provided hostGroup is present in it
2130             resp_xml = XmlElementTree.fromstring(resp.content)
2131             for child in resp_xml:
2132                 if 'VMWProviderVdcResourcePool' in child.tag:
2133                     for schild in child:
2134                         if 'Link' in schild.tag:
2135                             if schild.attrib.get('type') == "application/vnd.vmware.admin.vmwHostGroupsType+xml":
2136                                 hostGroup = schild.attrib.get('href')
2137                                 hg_resp = self.perform_request(req_type='GET',url=hostGroup, headers=headers)
2138                                 if hg_resp.status_code != requests.codes.ok:
2139                                     self.logger.debug ("REST API call {} failed. Return status code {}".format(hostGroup, hg_resp.status_code))
2140                                 else:
2141                                     hg_resp_xml =  XmlElementTree.fromstring(hg_resp.content)
2142                                     for hostGroup in hg_resp_xml:
2143                                         if 'HostGroup' in hostGroup.tag:
2144                                             #append host group name to the list
2145                                             vcd_az.append(hostGroup.attrib.get("name"))
2146         return vcd_az
2147
2148
2149     def set_availability_zones(self):
2150         """
2151         Set vim availability zone
2152         """
2153
2154         vim_availability_zones = None
2155         availability_zone = None
2156         if 'availability_zone' in self.config:
2157             vim_availability_zones = self.config.get('availability_zone')
2158         if isinstance(vim_availability_zones, str):
2159             availability_zone = [vim_availability_zones]
2160         elif isinstance(vim_availability_zones, list):
2161             availability_zone = vim_availability_zones
2162         else:
2163             return availability_zone
2164
2165         return availability_zone
2166
2167
2168     def get_vm_availability_zone(self, availability_zone_index, availability_zone_list):
2169         """
2170         Return the availability zone to be used by the created VM.
2171         returns: The VIM availability zone to be used or None
2172         """
2173         if availability_zone_index is None:
2174             if not self.config.get('availability_zone'):
2175                 return None
2176             elif isinstance(self.config.get('availability_zone'), str):
2177                 return self.config['availability_zone']
2178             else:
2179                 return self.config['availability_zone'][0]
2180
2181         vim_availability_zones = self.availability_zone
2182
2183         # check if VIM offer enough availability zones describe in the VNFD
2184         if vim_availability_zones and len(availability_zone_list) <= len(vim_availability_zones):
2185             # check if all the names of NFV AV match VIM AV names
2186             match_by_index = False
2187             for av in availability_zone_list:
2188                 if av not in vim_availability_zones:
2189                     match_by_index = True
2190                     break
2191             if match_by_index:
2192                 self.logger.debug("Required Availability zone or Host Group not found in VIM config")
2193                 self.logger.debug("Input Availability zone list: {}".format(availability_zone_list))
2194                 self.logger.debug("VIM configured Availability zones: {}".format(vim_availability_zones))
2195                 self.logger.debug("VIM Availability zones will be used by index")
2196                 return vim_availability_zones[availability_zone_index]
2197             else:
2198                 return availability_zone_list[availability_zone_index]
2199         else:
2200             raise vimconn.vimconnConflictException("No enough availability zones at VIM for this deployment")
2201
2202
2203     def create_vm_to_host_affinity_rule(self, addrule_href, vmgrpname, hostgrpname, polarity, headers):
2204         """ Method to create VM to Host Affinity rule in vCD
2205
2206         Args:
2207             addrule_href - href to make a POST request
2208             vmgrpname - name of the VM group created
2209             hostgrpnmae - name of the host group created earlier
2210             polarity - Affinity or Anti-affinity (default: Affinity)
2211             headers - headers to make REST call
2212
2213         Returns:
2214             True- if rule is created
2215             False- Failed to create rule due to some error
2216
2217         """
2218         task_status = False
2219         rule_name = polarity + "_" + vmgrpname
2220         payload = """<?xml version="1.0" encoding="UTF-8"?>
2221                      <vmext:VMWVmHostAffinityRule
2222                        xmlns:vmext="http://www.vmware.com/vcloud/extension/v1.5"
2223                        xmlns:vcloud="http://www.vmware.com/vcloud/v1.5"
2224                        type="application/vnd.vmware.admin.vmwVmHostAffinityRule+xml">
2225                        <vcloud:Name>{}</vcloud:Name>
2226                        <vcloud:IsEnabled>true</vcloud:IsEnabled>
2227                        <vcloud:IsMandatory>true</vcloud:IsMandatory>
2228                        <vcloud:Polarity>{}</vcloud:Polarity>
2229                        <vmext:HostGroupName>{}</vmext:HostGroupName>
2230                        <vmext:VmGroupName>{}</vmext:VmGroupName>
2231                      </vmext:VMWVmHostAffinityRule>""".format(rule_name, polarity, hostgrpname, vmgrpname)
2232
2233         resp = self.perform_request(req_type='POST',url=addrule_href, headers=headers, data=payload)
2234
2235         if resp.status_code != requests.codes.accepted:
2236             self.logger.debug ("REST API call {} failed. Return status code {}".format(addrule_href, resp.status_code))
2237             task_status = False
2238             return task_status
2239         else:
2240             affinity_task = self.get_task_from_response(resp.content)
2241             self.logger.debug ("affinity_task: {}".format(affinity_task))
2242             if affinity_task is None or affinity_task is False:
2243                 raise vimconn.vimconnUnexpectedResponse("failed to find affinity task")
2244             # wait for task to complete
2245             result = self.client.get_task_monitor().wait_for_success(task=affinity_task)
2246             if result.get('status') == 'success':
2247                 self.logger.debug("Successfully created affinity rule {}".format(rule_name))
2248                 return True
2249             else:
2250                 raise vimconn.vimconnUnexpectedResponse(
2251                       "failed to create affinity rule {}".format(rule_name))
2252
2253
2254     def get_add_rule_reference (self, respool_href, headers):
2255         """ This method finds href to add vm to host affinity rule to vCD
2256
2257         Args:
2258             respool_href- href to resource pool
2259             headers- header information to make REST call
2260
2261         Returns:
2262             None - if no valid href to add rule found or
2263             addrule_href - href to add vm to host affinity rule of resource pool
2264         """
2265         addrule_href = None
2266         resp = self.perform_request(req_type='GET',url=respool_href, headers=headers)
2267
2268         if resp.status_code != requests.codes.ok:
2269             self.logger.debug ("REST API call {} failed. Return status code {}".format(respool_href, resp.status_code))
2270         else:
2271
2272             resp_xml = XmlElementTree.fromstring(resp.content)
2273             for child in resp_xml:
2274                 if 'VMWProviderVdcResourcePool' in child.tag:
2275                     for schild in child:
2276                         if 'Link' in schild.tag:
2277                             if schild.attrib.get('type') == "application/vnd.vmware.admin.vmwVmHostAffinityRule+xml" and \
2278                                 schild.attrib.get('rel') == "add":
2279                                 addrule_href = schild.attrib.get('href')
2280                                 break
2281
2282         return addrule_href
2283
2284
2285     def add_vm_to_vmgroup(self, vm_uuid, vmGroupNameURL, vmGroup_name, headers):
2286         """ Method to add deployed VM to newly created VM Group.
2287             This is required to create VM to Host affinity in vCD
2288
2289         Args:
2290             vm_uuid- newly created vm uuid
2291             vmGroupNameURL- URL to VM Group name
2292             vmGroup_name- Name of VM group created
2293             headers- Headers for REST request
2294
2295         Returns:
2296             True- if VM added to VM group successfully
2297             False- if any error encounter
2298         """
2299
2300         addvm_resp = self.perform_request(req_type='GET',url=vmGroupNameURL, headers=headers)#, data=payload)
2301
2302         if addvm_resp.status_code != requests.codes.ok:
2303             self.logger.debug ("REST API call to get VM Group Name url {} failed. Return status code {}"\
2304                                .format(vmGroupNameURL, addvm_resp.status_code))
2305             return False
2306         else:
2307             resp_xml = XmlElementTree.fromstring(addvm_resp.content)
2308             for child in resp_xml:
2309                 if child.tag.split('}')[1] == 'Link':
2310                     if child.attrib.get("rel") == "addVms":
2311                         addvmtogrpURL =  child.attrib.get("href")
2312
2313         #Get vm details
2314         url_list = [self.url, '/api/vApp/vm-',vm_uuid]
2315         vmdetailsURL = ''.join(url_list)
2316
2317         resp = self.perform_request(req_type='GET',url=vmdetailsURL, headers=headers)
2318
2319         if resp.status_code != requests.codes.ok:
2320             self.logger.debug ("REST API call {} failed. Return status code {}".format(vmdetailsURL, resp.status_code))
2321             return False
2322
2323         #Parse VM details
2324         resp_xml = XmlElementTree.fromstring(resp.content)
2325         if resp_xml.tag.split('}')[1] == "Vm":
2326             vm_id = resp_xml.attrib.get("id")
2327             vm_name = resp_xml.attrib.get("name")
2328             vm_href = resp_xml.attrib.get("href")
2329             #print vm_id, vm_name, vm_href
2330         #Add VM into VMgroup
2331         payload = """<?xml version="1.0" encoding="UTF-8"?>\
2332                    <ns2:Vms xmlns:ns2="http://www.vmware.com/vcloud/v1.5" \
2333                     xmlns="http://www.vmware.com/vcloud/versions" \
2334                     xmlns:ns3="http://schemas.dmtf.org/ovf/envelope/1" \
2335                     xmlns:ns4="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData" \
2336                     xmlns:ns5="http://schemas.dmtf.org/wbem/wscim/1/common" \
2337                     xmlns:ns6="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData" \
2338                     xmlns:ns7="http://www.vmware.com/schema/ovf" \
2339                     xmlns:ns8="http://schemas.dmtf.org/ovf/environment/1" \
2340                     xmlns:ns9="http://www.vmware.com/vcloud/extension/v1.5">\
2341                     <ns2:VmReference href="{}" id="{}" name="{}" \
2342                     type="application/vnd.vmware.vcloud.vm+xml" />\
2343                    </ns2:Vms>""".format(vm_href, vm_id, vm_name)
2344
2345         addvmtogrp_resp = self.perform_request(req_type='POST',url=addvmtogrpURL, headers=headers, data=payload)
2346
2347         if addvmtogrp_resp.status_code != requests.codes.accepted:
2348             self.logger.debug ("REST API call {} failed. Return status code {}".format(addvmtogrpURL, addvmtogrp_resp.status_code))
2349             return False
2350         else:
2351             self.logger.debug ("Done adding VM {} to VMgroup {}".format(vm_name, vmGroup_name))
2352             return True
2353
2354
2355     def create_vmgroup(self, vmgroup_name, vmgroup_href, headers):
2356         """Method to create a VM group in vCD
2357
2358            Args:
2359               vmgroup_name : Name of VM group to be created
2360               vmgroup_href : href for vmgroup
2361               headers- Headers for REST request
2362         """
2363         #POST to add URL with required data
2364         vmgroup_status = False
2365         payload = """<VMWVmGroup xmlns="http://www.vmware.com/vcloud/extension/v1.5" \
2366                        xmlns:vcloud_v1.5="http://www.vmware.com/vcloud/v1.5" name="{}">\
2367                    <vmCount>1</vmCount>\
2368                    </VMWVmGroup>""".format(vmgroup_name)
2369         resp = self.perform_request(req_type='POST',url=vmgroup_href, headers=headers, data=payload)
2370
2371         if resp.status_code != requests.codes.accepted:
2372             self.logger.debug ("REST API call {} failed. Return status code {}".format(vmgroup_href, resp.status_code))
2373             return vmgroup_status
2374         else:
2375             vmgroup_task = self.get_task_from_response(resp.content)
2376             if vmgroup_task is None or vmgroup_task is False:
2377                 raise vimconn.vimconnUnexpectedResponse(
2378                     "create_vmgroup(): failed to create VM group {}".format(vmgroup_name))
2379
2380             # wait for task to complete
2381             result = self.client.get_task_monitor().wait_for_success(task=vmgroup_task)
2382
2383             if result.get('status') == 'success':
2384                 self.logger.debug("create_vmgroup(): Successfully created VM group {}".format(vmgroup_name))
2385                 #time.sleep(10)
2386                 vmgroup_status = True
2387                 return vmgroup_status
2388             else:
2389                 raise vimconn.vimconnUnexpectedResponse(\
2390                         "create_vmgroup(): failed to create VM group {}".format(vmgroup_name))
2391
2392
2393     def find_vmgroup_reference(self, url, headers):
2394         """ Method to create a new VMGroup which is required to add created VM
2395             Args:
2396                url- resource pool href
2397                headers- header information
2398
2399             Returns:
2400                returns href to VM group to create VM group
2401         """
2402         #Perform GET on resource pool to find 'add' link to create VMGroup
2403         #https://vcd-ip/api/admin/extension/providervdc/<providervdc id>/resourcePools
2404         vmgrp_href = None
2405         resp = self.perform_request(req_type='GET',url=url, headers=headers)
2406
2407         if resp.status_code != requests.codes.ok:
2408             self.logger.debug ("REST API call {} failed. Return status code {}".format(url, resp.status_code))
2409         else:
2410             #Get the href to add vmGroup to vCD
2411             resp_xml = XmlElementTree.fromstring(resp.content)
2412             for child in resp_xml:
2413                 if 'VMWProviderVdcResourcePool' in child.tag:
2414                     for schild in child:
2415                         if 'Link' in schild.tag:
2416                             #Find href with type VMGroup and rel with add
2417                             if schild.attrib.get('type') == "application/vnd.vmware.admin.vmwVmGroupType+xml"\
2418                                 and schild.attrib.get('rel') == "add":
2419                                 vmgrp_href = schild.attrib.get('href')
2420                                 return vmgrp_href
2421
2422
2423     def check_availibility_zone(self, az, respool_href, headers):
2424         """ Method to verify requested av zone is present or not in provided
2425             resource pool
2426
2427             Args:
2428                 az - name of hostgroup (availibility_zone)
2429                 respool_href - Resource Pool href
2430                 headers - Headers to make REST call
2431             Returns:
2432                 az_found - True if availibility_zone is found else False
2433         """
2434         az_found = False
2435         headers['Accept']='application/*+xml;version=27.0'
2436         resp = self.perform_request(req_type='GET',url=respool_href, headers=headers)
2437
2438         if resp.status_code != requests.codes.ok:
2439             self.logger.debug ("REST API call {} failed. Return status code {}".format(respool_href, resp.status_code))
2440         else:
2441         #Get the href to hostGroups and find provided hostGroup is present in it
2442             resp_xml = XmlElementTree.fromstring(resp.content)
2443
2444             for child in resp_xml:
2445                 if 'VMWProviderVdcResourcePool' in child.tag:
2446                     for schild in child:
2447                         if 'Link' in schild.tag:
2448                             if schild.attrib.get('type') == "application/vnd.vmware.admin.vmwHostGroupsType+xml":
2449                                 hostGroup_href = schild.attrib.get('href')
2450                                 hg_resp = self.perform_request(req_type='GET',url=hostGroup_href, headers=headers)
2451                                 if hg_resp.status_code != requests.codes.ok:
2452                                     self.logger.debug ("REST API call {} failed. Return status code {}".format(hostGroup_href, hg_resp.status_code))
2453                                 else:
2454                                     hg_resp_xml = XmlElementTree.fromstring(hg_resp.content)
2455                                     for hostGroup in hg_resp_xml:
2456                                         if 'HostGroup' in hostGroup.tag:
2457                                             if hostGroup.attrib.get("name") == az:
2458                                                 az_found = True
2459                                                 break
2460         return az_found
2461
2462
2463     def get_pvdc_for_org(self, org_vdc, headers):
2464         """ This method gets provider vdc references from organisation
2465
2466             Args:
2467                org_vdc - name of the organisation VDC to find pvdc
2468                headers - headers to make REST call
2469
2470             Returns:
2471                None - if no pvdc href found else
2472                pvdc_href - href to pvdc
2473         """
2474
2475         #Get provider VDC references from vCD
2476         pvdc_href = None
2477         #url = '<vcd url>/api/admin/extension/providerVdcReferences'
2478         url_list = [self.url, '/api/admin/extension/providerVdcReferences']
2479         url = ''.join(url_list)
2480
2481         response = self.perform_request(req_type='GET',url=url, headers=headers)
2482         if response.status_code != requests.codes.ok:
2483             self.logger.debug ("REST API call {} failed. Return status code {}"\
2484                                .format(url, response.status_code))
2485         else:
2486             xmlroot_response = XmlElementTree.fromstring(response.content)
2487             for child in xmlroot_response:
2488                 if 'ProviderVdcReference' in child.tag:
2489                     pvdc_href = child.attrib.get('href')
2490                     #Get vdcReferences to find org
2491                     pvdc_resp = self.perform_request(req_type='GET',url=pvdc_href, headers=headers)
2492                     if pvdc_resp.status_code != requests.codes.ok:
2493                         raise vimconn.vimconnException("REST API call {} failed. "\
2494                                                        "Return status code {}"\
2495                                                        .format(url, pvdc_resp.status_code))
2496
2497                     pvdc_resp_xml = XmlElementTree.fromstring(pvdc_resp.content)
2498                     for child in pvdc_resp_xml:
2499                         if 'Link' in child.tag:
2500                             if child.attrib.get('type') == "application/vnd.vmware.admin.vdcReferences+xml":
2501                                 vdc_href = child.attrib.get('href')
2502
2503                                 #Check if provided org is present in vdc
2504                                 vdc_resp = self.perform_request(req_type='GET',
2505                                                                 url=vdc_href,
2506                                                                 headers=headers)
2507                                 if vdc_resp.status_code != requests.codes.ok:
2508                                     raise vimconn.vimconnException("REST API call {} failed. "\
2509                                                                    "Return status code {}"\
2510                                                                    .format(url, vdc_resp.status_code))
2511                                 vdc_resp_xml = XmlElementTree.fromstring(vdc_resp.content)
2512                                 for child in vdc_resp_xml:
2513                                     if 'VdcReference' in child.tag:
2514                                         if child.attrib.get('name') == org_vdc:
2515                                             return pvdc_href
2516
2517
2518     def get_resource_pool_details(self, pvdc_href, headers):
2519         """ Method to get resource pool information.
2520             Host groups are property of resource group.
2521             To get host groups, we need to GET details of resource pool.
2522
2523             Args:
2524                 pvdc_href: href to pvdc details
2525                 headers: headers
2526
2527             Returns:
2528                 respool_href - Returns href link reference to resource pool
2529         """
2530         respool_href = None
2531         resp = self.perform_request(req_type='GET',url=pvdc_href, headers=headers)
2532
2533         if resp.status_code != requests.codes.ok:
2534             self.logger.debug ("REST API call {} failed. Return status code {}"\
2535                                .format(pvdc_href, resp.status_code))
2536         else:
2537             respool_resp_xml = XmlElementTree.fromstring(resp.content)
2538             for child in respool_resp_xml:
2539                 if 'Link' in child.tag:
2540                     if child.attrib.get('type') == "application/vnd.vmware.admin.vmwProviderVdcResourcePoolSet+xml":
2541                         respool_href = child.attrib.get("href")
2542                         break
2543         return respool_href
2544
2545
2546     def log_message(self, msg):
2547         """
2548             Method to log error messages related to Affinity rule creation
2549             in new_vminstance & raise Exception
2550                 Args :
2551                     msg - Error message to be logged
2552
2553         """
2554         #get token to connect vCD as a normal user
2555         self.get_token()
2556         self.logger.debug(msg)
2557         raise vimconn.vimconnException(msg)
2558
2559
2560     ##
2561     ##
2562     ##  based on current discussion
2563     ##
2564     ##
2565     ##  server:
2566     #   created: '2016-09-08T11:51:58'
2567     #   description: simple-instance.linux1.1
2568     #   flavor: ddc6776e-75a9-11e6-ad5f-0800273e724c
2569     #   hostId: e836c036-74e7-11e6-b249-0800273e724c
2570     #   image: dde30fe6-75a9-11e6-ad5f-0800273e724c
2571     #   status: ACTIVE
2572     #   error_msg:
2573     #   interfaces: …
2574     #
2575     def get_vminstance(self, vim_vm_uuid=None):
2576         """Returns the VM instance information from VIM"""
2577
2578         self.logger.debug("Client requesting vm instance {} ".format(vim_vm_uuid))
2579
2580         org, vdc = self.get_vdc_details()
2581         if vdc is None:
2582             raise vimconn.vimconnConnectionException(
2583                 "Failed to get a reference of VDC for a tenant {}".format(self.tenant_name))
2584
2585         vm_info_dict = self.get_vapp_details_rest(vapp_uuid=vim_vm_uuid)
2586         if not vm_info_dict:
2587             self.logger.debug("get_vminstance(): Failed to get vApp name by UUID {}".format(vim_vm_uuid))
2588             raise vimconn.vimconnNotFoundException("Failed to get vApp name by UUID {}".format(vim_vm_uuid))
2589
2590         status_key = vm_info_dict['status']
2591         error = ''
2592         try:
2593             vm_dict = {'created': vm_info_dict['created'],
2594                        'description': vm_info_dict['name'],
2595                        'status': vcdStatusCode2manoFormat[int(status_key)],
2596                        'hostId': vm_info_dict['vmuuid'],
2597                        'error_msg': error,
2598                        'vim_info': yaml.safe_dump(vm_info_dict), 'interfaces': []}
2599
2600             if 'interfaces' in vm_info_dict:
2601                 vm_dict['interfaces'] = vm_info_dict['interfaces']
2602             else:
2603                 vm_dict['interfaces'] = []
2604         except KeyError:
2605             vm_dict = {'created': '',
2606                        'description': '',
2607                        'status': vcdStatusCode2manoFormat[int(-1)],
2608                        'hostId': vm_info_dict['vmuuid'],
2609                        'error_msg': "Inconsistency state",
2610                        'vim_info': yaml.safe_dump(vm_info_dict), 'interfaces': []}
2611
2612         return vm_dict
2613
2614     def delete_vminstance(self, vm__vim_uuid, created_items=None):
2615         """Method poweroff and remove VM instance from vcloud director network.
2616
2617         Args:
2618             vm__vim_uuid: VM UUID
2619
2620         Returns:
2621             Returns the instance identifier
2622         """
2623
2624         self.logger.debug("Client requesting delete vm instance {} ".format(vm__vim_uuid))
2625
2626         org, vdc = self.get_vdc_details()
2627         vdc_obj = VDC(self.client, href=vdc.get('href'))
2628         if vdc_obj is None:
2629             self.logger.debug("delete_vminstance(): Failed to get a reference of VDC for a tenant {}".format(
2630                 self.tenant_name))
2631             raise vimconn.vimconnException(
2632                 "delete_vminstance(): Failed to get a reference of VDC for a tenant {}".format(self.tenant_name))
2633
2634         try:
2635             vapp_name = self.get_namebyvappid(vm__vim_uuid)
2636             vapp_resource = vdc_obj.get_vapp(vapp_name)
2637             vapp = VApp(self.client, resource=vapp_resource)
2638             if vapp_name is None:
2639                 self.logger.debug("delete_vminstance(): Failed to get vm by given {} vm uuid".format(vm__vim_uuid))
2640                 return -1, "delete_vminstance(): Failed to get vm by given {} vm uuid".format(vm__vim_uuid)
2641             else:
2642                 self.logger.info("Deleting vApp {} and UUID {}".format(vapp_name, vm__vim_uuid))
2643
2644             # Delete vApp and wait for status change if task executed and vApp is None.
2645
2646             if vapp:
2647                 if vapp_resource.get('deployed') == 'true':
2648                     self.logger.info("Powering off vApp {}".format(vapp_name))
2649                     #Power off vApp
2650                     powered_off = False
2651                     wait_time = 0
2652                     while wait_time <= MAX_WAIT_TIME:
2653                         power_off_task = vapp.power_off()
2654                         result = self.client.get_task_monitor().wait_for_success(task=power_off_task)
2655
2656                         if result.get('status') == 'success':
2657                             powered_off = True
2658                             break
2659                         else:
2660                             self.logger.info("Wait for vApp {} to power off".format(vapp_name))
2661                             time.sleep(INTERVAL_TIME)
2662
2663                         wait_time +=INTERVAL_TIME
2664                     if not powered_off:
2665                         self.logger.debug("delete_vminstance(): Failed to power off VM instance {} ".format(vm__vim_uuid))
2666                     else:
2667                         self.logger.info("delete_vminstance(): Powered off VM instance {} ".format(vm__vim_uuid))
2668
2669                     #Undeploy vApp
2670                     self.logger.info("Undeploy vApp {}".format(vapp_name))
2671                     wait_time = 0
2672                     undeployed = False
2673                     while wait_time <= MAX_WAIT_TIME:
2674                         vapp = VApp(self.client, resource=vapp_resource)
2675                         if not vapp:
2676                             self.logger.debug("delete_vminstance(): Failed to get vm by given {} vm uuid".format(vm__vim_uuid))
2677                             return -1, "delete_vminstance(): Failed to get vm by given {} vm uuid".format(vm__vim_uuid)
2678                         undeploy_task = vapp.undeploy()
2679
2680                         result = self.client.get_task_monitor().wait_for_success(task=undeploy_task)
2681                         if result.get('status') == 'success':
2682                             undeployed = True
2683                             break
2684                         else:
2685                             self.logger.debug("Wait for vApp {} to undeploy".format(vapp_name))
2686                             time.sleep(INTERVAL_TIME)
2687
2688                         wait_time +=INTERVAL_TIME
2689
2690                     if not undeployed:
2691                         self.logger.debug("delete_vminstance(): Failed to undeploy vApp {} ".format(vm__vim_uuid))
2692
2693                 # delete vapp
2694                 self.logger.info("Start deletion of vApp {} ".format(vapp_name))
2695
2696                 if vapp is not None:
2697                     wait_time = 0
2698                     result = False
2699
2700                     while wait_time <= MAX_WAIT_TIME:
2701                         vapp = VApp(self.client, resource=vapp_resource)
2702                         if not vapp:
2703                             self.logger.debug("delete_vminstance(): Failed to get vm by given {} vm uuid".format(vm__vim_uuid))
2704                             return -1, "delete_vminstance(): Failed to get vm by given {} vm uuid".format(vm__vim_uuid)
2705
2706                         delete_task = vdc_obj.delete_vapp(vapp.name, force=True)
2707
2708                         result = self.client.get_task_monitor().wait_for_success(task=delete_task)
2709                         if result.get('status') == 'success':
2710                             break
2711                         else:
2712                             self.logger.debug("Wait for vApp {} to delete".format(vapp_name))
2713                             time.sleep(INTERVAL_TIME)
2714
2715                         wait_time +=INTERVAL_TIME
2716
2717                     if result is None:
2718                         self.logger.debug("delete_vminstance(): Failed delete uuid {} ".format(vm__vim_uuid))
2719                     else:
2720                         self.logger.info("Deleted vm instance {} sccessfully".format(vm__vim_uuid))
2721                         return vm__vim_uuid
2722         except:
2723             self.logger.debug(traceback.format_exc())
2724             raise vimconn.vimconnException("delete_vminstance(): Failed delete vm instance {}".format(vm__vim_uuid))
2725
2726
2727     def refresh_vms_status(self, vm_list):
2728         """Get the status of the virtual machines and their interfaces/ports
2729            Params: the list of VM identifiers
2730            Returns a dictionary with:
2731                 vm_id:          #VIM id of this Virtual Machine
2732                     status:     #Mandatory. Text with one of:
2733                                 #  DELETED (not found at vim)
2734                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
2735                                 #  OTHER (Vim reported other status not understood)
2736                                 #  ERROR (VIM indicates an ERROR status)
2737                                 #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
2738                                 #  CREATING (on building process), ERROR
2739                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
2740                                 #
2741                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
2742                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
2743                     interfaces:
2744                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
2745                         mac_address:      #Text format XX:XX:XX:XX:XX:XX
2746                         vim_net_id:       #network id where this interface is connected
2747                         vim_interface_id: #interface/port VIM id
2748                         ip_address:       #null, or text with IPv4, IPv6 address
2749         """
2750
2751         self.logger.debug("Client requesting refresh vm status for {} ".format(vm_list))
2752
2753         org,vdc = self.get_vdc_details()
2754         if vdc is None:
2755             raise vimconn.vimconnException("Failed to get a reference of VDC for a tenant {}".format(self.tenant_name))
2756
2757         vms_dict = {}
2758         nsx_edge_list = []
2759         for vmuuid in vm_list:
2760             vapp_name = self.get_namebyvappid(vmuuid)
2761             if vapp_name is not None:
2762
2763                 try:
2764                     vm_pci_details = self.get_vm_pci_details(vmuuid)
2765                     vdc_obj = VDC(self.client, href=vdc.get('href'))
2766                     vapp_resource = vdc_obj.get_vapp(vapp_name)
2767                     the_vapp = VApp(self.client, resource=vapp_resource)
2768
2769                     vm_details = {}
2770                     for vm in the_vapp.get_all_vms():
2771                         headers = {'Accept':'application/*+xml;version=' + API_VERSION,
2772                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
2773                         response = self.perform_request(req_type='GET',
2774                                                         url=vm.get('href'),
2775                                                         headers=headers)
2776
2777                         if response.status_code != 200:
2778                             self.logger.error("refresh_vms_status : REST call {} failed reason : {}"\
2779                                                             "status code : {}".format(vm.get('href'),
2780                                                                                     response.content,
2781                                                                                response.status_code))
2782                             raise vimconn.vimconnException("refresh_vms_status : Failed to get "\
2783                                                                          "VM details")
2784                         xmlroot = XmlElementTree.fromstring(response.content)
2785
2786
2787                         result = response.content.replace("\n"," ")
2788                         hdd_match = re.search('vcloud:capacity="(\d+)"\svcloud:storageProfileOverrideVmDefault=',result)
2789                         if hdd_match:
2790                             hdd_mb = hdd_match.group(1)
2791                             vm_details['hdd_mb'] = int(hdd_mb) if hdd_mb else None
2792                         cpus_match = re.search('<rasd:Description>Number of Virtual CPUs</.*?>(\d+)</rasd:VirtualQuantity>',result)
2793                         if cpus_match:
2794                             cpus = cpus_match.group(1)
2795                             vm_details['cpus'] = int(cpus) if cpus else None
2796                         memory_mb = re.search('<rasd:Description>Memory Size</.*?>(\d+)</rasd:VirtualQuantity>',result).group(1)
2797                         vm_details['memory_mb'] = int(memory_mb) if memory_mb else None
2798                         vm_details['status'] = vcdStatusCode2manoFormat[int(xmlroot.get('status'))]
2799                         vm_details['id'] = xmlroot.get('id')
2800                         vm_details['name'] = xmlroot.get('name')
2801                         vm_info = [vm_details]
2802                         if vm_pci_details:
2803                             vm_info[0].update(vm_pci_details)
2804
2805                         vm_dict = {'status': vcdStatusCode2manoFormat[int(vapp_resource.get('status'))],
2806                                    'error_msg': vcdStatusCode2manoFormat[int(vapp_resource.get('status'))],
2807                                    'vim_info': yaml.safe_dump(vm_info), 'interfaces': []}
2808
2809                         # get networks
2810                         vm_ip = None
2811                         vm_mac = None
2812                         networks = re.findall('<NetworkConnection needsCustomization=.*?</NetworkConnection>',result)
2813                         for network in networks:
2814                             mac_s = re.search('<MACAddress>(.*?)</MACAddress>',network)
2815                             vm_mac = mac_s.group(1) if mac_s else None
2816                             ip_s = re.search('<IpAddress>(.*?)</IpAddress>',network)
2817                             vm_ip = ip_s.group(1) if ip_s else None
2818
2819                             if vm_ip is None:
2820                                 if not nsx_edge_list:
2821                                     nsx_edge_list = self.get_edge_details()
2822                                     if nsx_edge_list is None:
2823                                         raise vimconn.vimconnException("refresh_vms_status:"\
2824                                                                        "Failed to get edge details from NSX Manager")
2825                                 if vm_mac is not None:
2826                                     vm_ip = self.get_ipaddr_from_NSXedge(nsx_edge_list, vm_mac)
2827
2828                             net_s = re.search('network="(.*?)"',network)
2829                             network_name = net_s.group(1) if net_s else None
2830
2831                             vm_net_id = self.get_network_id_by_name(network_name)
2832                             interface = {"mac_address": vm_mac,
2833                                          "vim_net_id": vm_net_id,
2834                                          "vim_interface_id": vm_net_id,
2835                                          "ip_address": vm_ip}
2836
2837                             vm_dict["interfaces"].append(interface)
2838
2839                     # add a vm to vm dict
2840                     vms_dict.setdefault(vmuuid, vm_dict)
2841                     self.logger.debug("refresh_vms_status : vm info {}".format(vm_dict))
2842                 except Exception as exp:
2843                     self.logger.debug("Error in response {}".format(exp))
2844                     self.logger.debug(traceback.format_exc())
2845
2846         return vms_dict
2847
2848
2849     def get_edge_details(self):
2850         """Get the NSX edge list from NSX Manager
2851            Returns list of NSX edges
2852         """
2853         edge_list = []
2854         rheaders = {'Content-Type': 'application/xml'}
2855         nsx_api_url = '/api/4.0/edges'
2856
2857         self.logger.debug("Get edge details from NSX Manager {} {}".format(self.nsx_manager, nsx_api_url))
2858
2859         try:
2860             resp = requests.get(self.nsx_manager + nsx_api_url,
2861                                 auth = (self.nsx_user, self.nsx_password),
2862                                 verify = False, headers = rheaders)
2863             if resp.status_code == requests.codes.ok:
2864                 paged_Edge_List = XmlElementTree.fromstring(resp.text)
2865                 for edge_pages in paged_Edge_List:
2866                     if edge_pages.tag == 'edgePage':
2867                         for edge_summary in edge_pages:
2868                             if edge_summary.tag == 'pagingInfo':
2869                                 for element in edge_summary:
2870                                     if element.tag == 'totalCount' and element.text == '0':
2871                                         raise vimconn.vimconnException("get_edge_details: No NSX edges details found: {}"
2872                                                                        .format(self.nsx_manager))
2873
2874                             if edge_summary.tag == 'edgeSummary':
2875                                 for element in edge_summary:
2876                                     if element.tag == 'id':
2877                                         edge_list.append(element.text)
2878                     else:
2879                         raise vimconn.vimconnException("get_edge_details: No NSX edge details found: {}"
2880                                                        .format(self.nsx_manager))
2881
2882                 if not edge_list:
2883                     raise vimconn.vimconnException("get_edge_details: "\
2884                                                    "No NSX edge details found: {}"
2885                                                    .format(self.nsx_manager))
2886                 else:
2887                     self.logger.debug("get_edge_details: Found NSX edges {}".format(edge_list))
2888                     return edge_list
2889             else:
2890                 self.logger.debug("get_edge_details: "
2891                                   "Failed to get NSX edge details from NSX Manager: {}"
2892                                   .format(resp.content))
2893                 return None
2894
2895         except Exception as exp:
2896             self.logger.debug("get_edge_details: "\
2897                               "Failed to get NSX edge details from NSX Manager: {}"
2898                               .format(exp))
2899             raise vimconn.vimconnException("get_edge_details: "\
2900                                            "Failed to get NSX edge details from NSX Manager: {}"
2901                                            .format(exp))
2902
2903
2904     def get_ipaddr_from_NSXedge(self, nsx_edges, mac_address):
2905         """Get IP address details from NSX edges, using the MAC address
2906            PARAMS: nsx_edges : List of NSX edges
2907                    mac_address : Find IP address corresponding to this MAC address
2908            Returns: IP address corrresponding to the provided MAC address
2909         """
2910
2911         ip_addr = None
2912         rheaders = {'Content-Type': 'application/xml'}
2913
2914         self.logger.debug("get_ipaddr_from_NSXedge: Finding IP addr from NSX edge")
2915
2916         try:
2917             for edge in nsx_edges:
2918                 nsx_api_url = '/api/4.0/edges/'+ edge +'/dhcp/leaseInfo'
2919
2920                 resp = requests.get(self.nsx_manager + nsx_api_url,
2921                                     auth = (self.nsx_user, self.nsx_password),
2922                                     verify = False, headers = rheaders)
2923
2924                 if resp.status_code == requests.codes.ok:
2925                     dhcp_leases = XmlElementTree.fromstring(resp.text)
2926                     for child in dhcp_leases:
2927                         if child.tag == 'dhcpLeaseInfo':
2928                             dhcpLeaseInfo = child
2929                             for leaseInfo in dhcpLeaseInfo:
2930                                 for elem in leaseInfo:
2931                                     if (elem.tag)=='macAddress':
2932                                         edge_mac_addr = elem.text
2933                                     if (elem.tag)=='ipAddress':
2934                                         ip_addr = elem.text
2935                                 if edge_mac_addr is not None:
2936                                     if edge_mac_addr == mac_address:
2937                                         self.logger.debug("Found ip addr {} for mac {} at NSX edge {}"
2938                                                           .format(ip_addr, mac_address,edge))
2939                                         return ip_addr
2940                 else:
2941                     self.logger.debug("get_ipaddr_from_NSXedge: "\
2942                                       "Error occurred while getting DHCP lease info from NSX Manager: {}"
2943                                       .format(resp.content))
2944
2945             self.logger.debug("get_ipaddr_from_NSXedge: No IP addr found in any NSX edge")
2946             return None
2947
2948         except XmlElementTree.ParseError as Err:
2949             self.logger.debug("ParseError in response from NSX Manager {}".format(Err.message), exc_info=True)
2950
2951
2952     def action_vminstance(self, vm__vim_uuid=None, action_dict=None, created_items={}):
2953         """Send and action over a VM instance from VIM
2954         Returns the vm_id if the action was successfully sent to the VIM"""
2955
2956         self.logger.debug("Received action for vm {} and action dict {}".format(vm__vim_uuid, action_dict))
2957         if vm__vim_uuid is None or action_dict is None:
2958             raise vimconn.vimconnException("Invalid request. VM id or action is None.")
2959
2960         org, vdc = self.get_vdc_details()
2961         if vdc is None:
2962             raise  vimconn.vimconnException("Failed to get a reference of VDC for a tenant {}".format(self.tenant_name))
2963
2964         vapp_name = self.get_namebyvappid(vm__vim_uuid)
2965         if vapp_name is None:
2966             self.logger.debug("action_vminstance(): Failed to get vm by given {} vm uuid".format(vm__vim_uuid))
2967             raise vimconn.vimconnException("Failed to get vm by given {} vm uuid".format(vm__vim_uuid))
2968         else:
2969             self.logger.info("Action_vminstance vApp {} and UUID {}".format(vapp_name, vm__vim_uuid))
2970
2971         try:
2972             vdc_obj = VDC(self.client, href=vdc.get('href'))
2973             vapp_resource = vdc_obj.get_vapp(vapp_name)
2974             vapp = VApp(self.client, resource=vapp_resource)
2975             if "start" in action_dict:
2976                 self.logger.info("action_vminstance: Power on vApp: {}".format(vapp_name))
2977                 poweron_task = self.power_on_vapp(vm__vim_uuid, vapp_name)
2978                 result = self.client.get_task_monitor().wait_for_success(task=poweron_task)
2979                 self.instance_actions_result("start", result, vapp_name)
2980             elif "rebuild" in action_dict:
2981                 self.logger.info("action_vminstance: Rebuild vApp: {}".format(vapp_name))
2982                 rebuild_task = vapp.deploy(power_on=True)
2983                 result = self.client.get_task_monitor().wait_for_success(task=rebuild_task)
2984                 self.instance_actions_result("rebuild", result, vapp_name)
2985             elif "pause" in action_dict:
2986                 self.logger.info("action_vminstance: pause vApp: {}".format(vapp_name))
2987                 pause_task = vapp.undeploy(action='suspend')
2988                 result = self.client.get_task_monitor().wait_for_success(task=pause_task)
2989                 self.instance_actions_result("pause", result, vapp_name)
2990             elif "resume" in action_dict:
2991                 self.logger.info("action_vminstance: resume vApp: {}".format(vapp_name))
2992                 poweron_task = self.power_on_vapp(vm__vim_uuid, vapp_name)
2993                 result = self.client.get_task_monitor().wait_for_success(task=poweron_task)
2994                 self.instance_actions_result("resume", result, vapp_name)
2995             elif "shutoff" in action_dict or "shutdown" in action_dict:
2996                 action_name , value = action_dict.items()[0]
2997                 #For python3
2998                 #action_name , value = list(action_dict.items())[0]
2999                 self.logger.info("action_vminstance: {} vApp: {}".format(action_name, vapp_name))
3000                 shutdown_task = vapp.shutdown()
3001                 result = self.client.get_task_monitor().wait_for_success(task=shutdown_task)
3002                 if action_name == "shutdown":
3003                     self.instance_actions_result("shutdown", result, vapp_name)
3004                 else:
3005                     self.instance_actions_result("shutoff", result, vapp_name)
3006             elif "forceOff" in action_dict:
3007                 result = vapp.undeploy(action='powerOff')
3008                 self.instance_actions_result("forceOff", result, vapp_name)
3009             elif "reboot" in action_dict:
3010                 self.logger.info("action_vminstance: reboot vApp: {}".format(vapp_name))
3011                 reboot_task = vapp.reboot()
3012                 self.client.get_task_monitor().wait_for_success(task=reboot_task)
3013             else:
3014                 raise vimconn.vimconnException("action_vminstance: Invalid action {} or action is None.".format(action_dict))
3015             return vm__vim_uuid
3016         except Exception as exp :
3017             self.logger.debug("action_vminstance: Failed with Exception {}".format(exp))
3018             raise vimconn.vimconnException("action_vminstance: Failed with Exception {}".format(exp))
3019
3020     def instance_actions_result(self, action, result, vapp_name):
3021         if result.get('status') == 'success':
3022             self.logger.info("action_vminstance: Sucessfully {} the vApp: {}".format(action, vapp_name))
3023         else:
3024             self.logger.error("action_vminstance: Failed to {} vApp: {}".format(action, vapp_name))
3025
3026     def get_vminstance_console(self, vm_id, console_type="vnc"):
3027         """
3028         Get a console for the virtual machine
3029         Params:
3030             vm_id: uuid of the VM
3031             console_type, can be:
3032                 "novnc" (by default), "xvpvnc" for VNC types,
3033                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
3034         Returns dict with the console parameters:
3035                 protocol: ssh, ftp, http, https, ...
3036                 server:   usually ip address
3037                 port:     the http, ssh, ... port
3038                 suffix:   extra text, e.g. the http path and query string
3039         """
3040         raise vimconn.vimconnNotImplemented("Should have implemented this")
3041
3042     # NOT USED METHODS in current version
3043
3044     def host_vim2gui(self, host, server_dict):
3045         """Transform host dictionary from VIM format to GUI format,
3046         and append to the server_dict
3047         """
3048         raise vimconn.vimconnNotImplemented("Should have implemented this")
3049
3050     def get_hosts_info(self):
3051         """Get the information of deployed hosts
3052         Returns the hosts content"""
3053         raise vimconn.vimconnNotImplemented("Should have implemented this")
3054
3055     def get_hosts(self, vim_tenant):
3056         """Get the hosts and deployed instances
3057         Returns the hosts content"""
3058         raise vimconn.vimconnNotImplemented("Should have implemented this")
3059
3060     def get_processor_rankings(self):
3061         """Get the processor rankings in the VIM database"""
3062         raise vimconn.vimconnNotImplemented("Should have implemented this")
3063
3064     def new_host(self, host_data):
3065         """Adds a new host to VIM"""
3066         '''Returns status code of the VIM response'''
3067         raise vimconn.vimconnNotImplemented("Should have implemented this")
3068
3069     def new_external_port(self, port_data):
3070         """Adds a external port to VIM"""
3071         '''Returns the port identifier'''
3072         raise vimconn.vimconnNotImplemented("Should have implemented this")
3073
3074     def new_external_network(self, net_name, net_type):
3075         """Adds a external network to VIM (shared)"""
3076         '''Returns the network identifier'''
3077         raise vimconn.vimconnNotImplemented("Should have implemented this")
3078
3079     def connect_port_network(self, port_id, network_id, admin=False):
3080         """Connects a external port to a network"""
3081         '''Returns status code of the VIM response'''
3082         raise vimconn.vimconnNotImplemented("Should have implemented this")
3083
3084     def new_vminstancefromJSON(self, vm_data):
3085         """Adds a VM instance to VIM"""
3086         '''Returns the instance identifier'''
3087         raise vimconn.vimconnNotImplemented("Should have implemented this")
3088
3089     def get_network_name_by_id(self, network_uuid=None):
3090         """Method gets vcloud director network named based on supplied uuid.
3091
3092         Args:
3093             network_uuid: network_id
3094
3095         Returns:
3096             The return network name.
3097         """
3098
3099         if not network_uuid:
3100             return None
3101
3102         try:
3103             org_dict = self.get_org(self.org_uuid)
3104             if 'networks' in org_dict:
3105                 org_network_dict = org_dict['networks']
3106                 for net_uuid in org_network_dict:
3107                     if net_uuid == network_uuid:
3108                         return org_network_dict[net_uuid]
3109         except:
3110             self.logger.debug("Exception in get_network_name_by_id")
3111             self.logger.debug(traceback.format_exc())
3112
3113         return None
3114
3115     def get_network_id_by_name(self, network_name=None):
3116         """Method gets vcloud director network uuid based on supplied name.
3117
3118         Args:
3119             network_name: network_name
3120         Returns:
3121             The return network uuid.
3122             network_uuid: network_id
3123         """
3124
3125         if not network_name:
3126             self.logger.debug("get_network_id_by_name() : Network name is empty")
3127             return None
3128
3129         try:
3130             org_dict = self.get_org(self.org_uuid)
3131             if org_dict and 'networks' in org_dict:
3132                 org_network_dict = org_dict['networks']
3133                 for net_uuid,net_name in org_network_dict.iteritems():
3134                 #For python3
3135                 #for net_uuid,net_name in org_network_dict.items():
3136                     if net_name == network_name:
3137                         return net_uuid
3138
3139         except KeyError as exp:
3140             self.logger.debug("get_network_id_by_name() : KeyError- {} ".format(exp))
3141
3142         return None
3143
3144     def list_org_action(self):
3145         """
3146         Method leverages vCloud director and query for available organization for particular user
3147
3148         Args:
3149             vca - is active VCA connection.
3150             vdc_name - is a vdc name that will be used to query vms action
3151
3152             Returns:
3153                 The return XML respond
3154         """
3155         url_list = [self.url, '/api/org']
3156         vm_list_rest_call = ''.join(url_list)
3157
3158         if self.client._session:
3159             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
3160                        'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
3161
3162             response = self.perform_request(req_type='GET',
3163                                      url=vm_list_rest_call,
3164                                            headers=headers)
3165
3166             if response.status_code == 403:
3167                 response = self.retry_rest('GET', vm_list_rest_call)
3168
3169             if response.status_code == requests.codes.ok:
3170                 return response.content
3171
3172         return None
3173
3174     def get_org_action(self, org_uuid=None):
3175         """
3176         Method leverages vCloud director and retrieve available object for organization.
3177
3178         Args:
3179             org_uuid - vCD organization uuid
3180             self.client - is active connection.
3181
3182             Returns:
3183                 The return XML respond
3184         """
3185
3186         if org_uuid is None:
3187             return None
3188
3189         url_list = [self.url, '/api/org/', org_uuid]
3190         vm_list_rest_call = ''.join(url_list)
3191
3192         if self.client._session:
3193             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
3194                      'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
3195
3196             #response = requests.get(vm_list_rest_call, headers=headers, verify=False)
3197             response = self.perform_request(req_type='GET',
3198                                             url=vm_list_rest_call,
3199                                             headers=headers)
3200             if response.status_code == 403:
3201                 response = self.retry_rest('GET', vm_list_rest_call)
3202
3203             if response.status_code == requests.codes.ok:
3204                 return response.content
3205         return None
3206
3207     def get_org(self, org_uuid=None):
3208         """
3209         Method retrieves available organization in vCloud Director
3210
3211         Args:
3212             org_uuid - is a organization uuid.
3213
3214             Returns:
3215                 The return dictionary with following key
3216                     "network" - for network list under the org
3217                     "catalogs" - for network list under the org
3218                     "vdcs" - for vdc list under org
3219         """
3220
3221         org_dict = {}
3222
3223         if org_uuid is None:
3224             return org_dict
3225
3226         content = self.get_org_action(org_uuid=org_uuid)
3227         try:
3228             vdc_list = {}
3229             network_list = {}
3230             catalog_list = {}
3231             vm_list_xmlroot = XmlElementTree.fromstring(content)
3232             for child in vm_list_xmlroot:
3233                 if child.attrib['type'] == 'application/vnd.vmware.vcloud.vdc+xml':
3234                     vdc_list[child.attrib['href'].split("/")[-1:][0]] = child.attrib['name']
3235                     org_dict['vdcs'] = vdc_list
3236                 if child.attrib['type'] == 'application/vnd.vmware.vcloud.orgNetwork+xml':
3237                     network_list[child.attrib['href'].split("/")[-1:][0]] = child.attrib['name']
3238                     org_dict['networks'] = network_list
3239                 if child.attrib['type'] == 'application/vnd.vmware.vcloud.catalog+xml':
3240                     catalog_list[child.attrib['href'].split("/")[-1:][0]] = child.attrib['name']
3241                     org_dict['catalogs'] = catalog_list
3242         except:
3243             pass
3244
3245         return org_dict
3246
3247     def get_org_list(self):
3248         """
3249         Method retrieves available organization in vCloud Director
3250
3251         Args:
3252             vca - is active VCA connection.
3253
3254             Returns:
3255                 The return dictionary and key for each entry VDC UUID
3256         """
3257
3258         org_dict = {}
3259
3260         content = self.list_org_action()
3261         try:
3262             vm_list_xmlroot = XmlElementTree.fromstring(content)
3263             for vm_xml in vm_list_xmlroot:
3264                 if vm_xml.tag.split("}")[1] == 'Org':
3265                     org_uuid = vm_xml.attrib['href'].split('/')[-1:]
3266                     org_dict[org_uuid[0]] = vm_xml.attrib['name']
3267         except:
3268             pass
3269
3270         return org_dict
3271
3272     def vms_view_action(self, vdc_name=None):
3273         """ Method leverages vCloud director vms query call
3274
3275         Args:
3276             vca - is active VCA connection.
3277             vdc_name - is a vdc name that will be used to query vms action
3278
3279             Returns:
3280                 The return XML respond
3281         """
3282         vca = self.connect()
3283         if vdc_name is None:
3284             return None
3285
3286         url_list = [vca.host, '/api/vms/query']
3287         vm_list_rest_call = ''.join(url_list)
3288
3289         if not (not vca.vcloud_session or not vca.vcloud_session.organization):
3290             refs = filter(lambda ref: ref.name == vdc_name and ref.type_ == 'application/vnd.vmware.vcloud.vdc+xml',
3291                           vca.vcloud_session.organization.Link)
3292             #For python3
3293             #refs = [ref for ref in vca.vcloud_session.organization.Link if ref.name == vdc_name and\
3294             #        ref.type_ == 'application/vnd.vmware.vcloud.vdc+xml']
3295             if len(refs) == 1:
3296                 response = Http.get(url=vm_list_rest_call,
3297                                     headers=vca.vcloud_session.get_vcloud_headers(),
3298                                     verify=vca.verify,
3299                                     logger=vca.logger)
3300                 if response.status_code == requests.codes.ok:
3301                     return response.content
3302
3303         return None
3304
3305     def get_vapp_list(self, vdc_name=None):
3306         """
3307         Method retrieves vApp list deployed vCloud director and returns a dictionary
3308         contains a list of all vapp deployed for queried VDC.
3309         The key for a dictionary is vApp UUID
3310
3311
3312         Args:
3313             vca - is active VCA connection.
3314             vdc_name - is a vdc name that will be used to query vms action
3315
3316             Returns:
3317                 The return dictionary and key for each entry vapp UUID
3318         """
3319
3320         vapp_dict = {}
3321         if vdc_name is None:
3322             return vapp_dict
3323
3324         content = self.vms_view_action(vdc_name=vdc_name)
3325         try:
3326             vm_list_xmlroot = XmlElementTree.fromstring(content)
3327             for vm_xml in vm_list_xmlroot:
3328                 if vm_xml.tag.split("}")[1] == 'VMRecord':
3329                     if vm_xml.attrib['isVAppTemplate'] == 'true':
3330                         rawuuid = vm_xml.attrib['container'].split('/')[-1:]
3331                         if 'vappTemplate-' in rawuuid[0]:
3332                             # vm in format vappTemplate-e63d40e7-4ff5-4c6d-851f-96c1e4da86a5 we remove
3333                             # vm and use raw UUID as key
3334                             vapp_dict[rawuuid[0][13:]] = vm_xml.attrib
3335         except:
3336             pass
3337
3338         return vapp_dict
3339
3340     def get_vm_list(self, vdc_name=None):
3341         """
3342         Method retrieves VM's list deployed vCloud director. It returns a dictionary
3343         contains a list of all VM's deployed for queried VDC.
3344         The key for a dictionary is VM UUID
3345
3346
3347         Args:
3348             vca - is active VCA connection.
3349             vdc_name - is a vdc name that will be used to query vms action
3350
3351             Returns:
3352                 The return dictionary and key for each entry vapp UUID
3353         """
3354         vm_dict = {}
3355
3356         if vdc_name is None:
3357             return vm_dict
3358
3359         content = self.vms_view_action(vdc_name=vdc_name)
3360         try:
3361             vm_list_xmlroot = XmlElementTree.fromstring(content)
3362             for vm_xml in vm_list_xmlroot:
3363                 if vm_xml.tag.split("}")[1] == 'VMRecord':
3364                     if vm_xml.attrib['isVAppTemplate'] == 'false':
3365                         rawuuid = vm_xml.attrib['href'].split('/')[-1:]
3366                         if 'vm-' in rawuuid[0]:
3367                             # vm in format vm-e63d40e7-4ff5-4c6d-851f-96c1e4da86a5 we remove
3368                             #  vm and use raw UUID as key
3369                             vm_dict[rawuuid[0][3:]] = vm_xml.attrib
3370         except:
3371             pass
3372
3373         return vm_dict
3374
3375     def get_vapp(self, vdc_name=None, vapp_name=None, isuuid=False):
3376         """
3377         Method retrieves VM deployed vCloud director. It returns VM attribute as dictionary
3378         contains a list of all VM's deployed for queried VDC.
3379         The key for a dictionary is VM UUID
3380
3381
3382         Args:
3383             vca - is active VCA connection.
3384             vdc_name - is a vdc name that will be used to query vms action
3385
3386             Returns:
3387                 The return dictionary and key for each entry vapp UUID
3388         """
3389         vm_dict = {}
3390         vca = self.connect()
3391         if not vca:
3392             raise vimconn.vimconnConnectionException("self.connect() is failed")
3393
3394         if vdc_name is None:
3395             return vm_dict
3396
3397         content = self.vms_view_action(vdc_name=vdc_name)
3398         try:
3399             vm_list_xmlroot = XmlElementTree.fromstring(content)
3400             for vm_xml in vm_list_xmlroot:
3401                 if vm_xml.tag.split("}")[1] == 'VMRecord' and vm_xml.attrib['isVAppTemplate'] == 'false':
3402                     # lookup done by UUID
3403                     if isuuid:
3404                         if vapp_name in vm_xml.attrib['container']:
3405                             rawuuid = vm_xml.attrib['href'].split('/')[-1:]
3406                             if 'vm-' in rawuuid[0]:
3407                                 vm_dict[rawuuid[0][3:]] = vm_xml.attrib
3408                                 break
3409                     # lookup done by Name
3410                     else:
3411                         if vapp_name in vm_xml.attrib['name']:
3412                             rawuuid = vm_xml.attrib['href'].split('/')[-1:]
3413                             if 'vm-' in rawuuid[0]:
3414                                 vm_dict[rawuuid[0][3:]] = vm_xml.attrib
3415                                 break
3416         except:
3417             pass
3418
3419         return vm_dict
3420
3421     def get_network_action(self, network_uuid=None):
3422         """
3423         Method leverages vCloud director and query network based on network uuid
3424
3425         Args:
3426             vca - is active VCA connection.
3427             network_uuid - is a network uuid
3428
3429             Returns:
3430                 The return XML respond
3431         """
3432
3433         if network_uuid is None:
3434             return None
3435
3436         url_list = [self.url, '/api/network/', network_uuid]
3437         vm_list_rest_call = ''.join(url_list)
3438
3439         if self.client._session:
3440             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
3441                      'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
3442
3443             response = self.perform_request(req_type='GET',
3444                                             url=vm_list_rest_call,
3445                                             headers=headers)
3446             #Retry login if session expired & retry sending request
3447             if response.status_code == 403:
3448                 response = self.retry_rest('GET', vm_list_rest_call)
3449
3450             if response.status_code == requests.codes.ok:
3451                 return response.content
3452
3453         return None
3454
3455     def get_vcd_network(self, network_uuid=None):
3456         """
3457         Method retrieves available network from vCloud Director
3458
3459         Args:
3460             network_uuid - is VCD network UUID
3461
3462         Each element serialized as key : value pair
3463
3464         Following keys available for access.    network_configuration['Gateway'}
3465         <Configuration>
3466           <IpScopes>
3467             <IpScope>
3468                 <IsInherited>true</IsInherited>
3469                 <Gateway>172.16.252.100</Gateway>
3470                 <Netmask>255.255.255.0</Netmask>
3471                 <Dns1>172.16.254.201</Dns1>
3472                 <Dns2>172.16.254.202</Dns2>
3473                 <DnsSuffix>vmwarelab.edu</DnsSuffix>
3474                 <IsEnabled>true</IsEnabled>
3475                 <IpRanges>
3476                     <IpRange>
3477                         <StartAddress>172.16.252.1</StartAddress>
3478                         <EndAddress>172.16.252.99</EndAddress>
3479                     </IpRange>
3480                 </IpRanges>
3481             </IpScope>
3482         </IpScopes>
3483         <FenceMode>bridged</FenceMode>
3484
3485         Returns:
3486                 The return dictionary and key for each entry vapp UUID
3487         """
3488
3489         network_configuration = {}
3490         if network_uuid is None:
3491             return network_uuid
3492
3493         try:
3494             content = self.get_network_action(network_uuid=network_uuid)
3495             vm_list_xmlroot = XmlElementTree.fromstring(content)
3496
3497             network_configuration['status'] = vm_list_xmlroot.get("status")
3498             network_configuration['name'] = vm_list_xmlroot.get("name")
3499             network_configuration['uuid'] = vm_list_xmlroot.get("id").split(":")[3]
3500
3501             for child in vm_list_xmlroot:
3502                 if child.tag.split("}")[1] == 'IsShared':
3503                     network_configuration['isShared'] = child.text.strip()
3504                 if child.tag.split("}")[1] == 'Configuration':
3505                     for configuration in child.iter():
3506                         tagKey = configuration.tag.split("}")[1].strip()
3507                         if tagKey != "":
3508                             network_configuration[tagKey] = configuration.text.strip()
3509             return network_configuration
3510         except Exception as exp :
3511             self.logger.debug("get_vcd_network: Failed with Exception {}".format(exp))
3512             raise vimconn.vimconnException("get_vcd_network: Failed with Exception {}".format(exp))
3513
3514         return network_configuration
3515
3516     def delete_network_action(self, network_uuid=None):
3517         """
3518         Method delete given network from vCloud director
3519
3520         Args:
3521             network_uuid - is a network uuid that client wish to delete
3522
3523             Returns:
3524                 The return None or XML respond or false
3525         """
3526         client = self.connect_as_admin()
3527         if not client:
3528             raise vimconn.vimconnConnectionException("Failed to connect vCD as admin")
3529         if network_uuid is None:
3530             return False
3531
3532         url_list = [self.url, '/api/admin/network/', network_uuid]
3533         vm_list_rest_call = ''.join(url_list)
3534
3535         if client._session:
3536             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
3537                      'x-vcloud-authorization': client._session.headers['x-vcloud-authorization']}
3538             response = self.perform_request(req_type='DELETE',
3539                                             url=vm_list_rest_call,
3540                                             headers=headers)
3541             if response.status_code == 202:
3542                 return True
3543
3544         return False
3545
3546     def create_network(self, network_name=None, net_type='bridge', parent_network_uuid=None,
3547                        ip_profile=None, isshared='true'):
3548         """
3549         Method create network in vCloud director
3550
3551         Args:
3552             network_name - is network name to be created.
3553             net_type - can be 'bridge','data','ptp','mgmt'.
3554             ip_profile is a dict containing the IP parameters of the network
3555             isshared - is a boolean
3556             parent_network_uuid - is parent provider vdc network that will be used for mapping.
3557             It optional attribute. by default if no parent network indicate the first available will be used.
3558
3559             Returns:
3560                 The return network uuid or return None
3561         """
3562
3563         new_network_name = [network_name, '-', str(uuid.uuid4())]
3564         content = self.create_network_rest(network_name=''.join(new_network_name),
3565                                            ip_profile=ip_profile,
3566                                            net_type=net_type,
3567                                            parent_network_uuid=parent_network_uuid,
3568                                            isshared=isshared)
3569         if content is None:
3570             self.logger.debug("Failed create network {}.".format(network_name))
3571             return None
3572
3573         try:
3574             vm_list_xmlroot = XmlElementTree.fromstring(content)
3575             vcd_uuid = vm_list_xmlroot.get('id').split(":")
3576             if len(vcd_uuid) == 4:
3577                 self.logger.info("Created new network name: {} uuid: {}".format(network_name, vcd_uuid[3]))
3578                 return vcd_uuid[3]
3579         except:
3580             self.logger.debug("Failed create network {}".format(network_name))
3581             return None
3582
3583     def create_network_rest(self, network_name=None, net_type='bridge', parent_network_uuid=None,
3584                             ip_profile=None, isshared='true'):
3585         """
3586         Method create network in vCloud director
3587
3588         Args:
3589             network_name - is network name to be created.
3590             net_type - can be 'bridge','data','ptp','mgmt'.
3591             ip_profile is a dict containing the IP parameters of the network
3592             isshared - is a boolean
3593             parent_network_uuid - is parent provider vdc network that will be used for mapping.
3594             It optional attribute. by default if no parent network indicate the first available will be used.
3595
3596             Returns:
3597                 The return network uuid or return None
3598         """
3599         client_as_admin = self.connect_as_admin()
3600         if not client_as_admin:
3601             raise vimconn.vimconnConnectionException("Failed to connect vCD.")
3602         if network_name is None:
3603             return None
3604
3605         url_list = [self.url, '/api/admin/vdc/', self.tenant_id]
3606         vm_list_rest_call = ''.join(url_list)
3607
3608         if client_as_admin._session:
3609             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
3610                      'x-vcloud-authorization': client_as_admin._session.headers['x-vcloud-authorization']}
3611
3612             response = self.perform_request(req_type='GET',
3613                                             url=vm_list_rest_call,
3614                                             headers=headers)
3615
3616             provider_network = None
3617             available_networks = None
3618             add_vdc_rest_url = None
3619
3620             if response.status_code != requests.codes.ok:
3621                 self.logger.debug("REST API call {} failed. Return status code {}".format(vm_list_rest_call,
3622                                                                                           response.status_code))
3623                 return None
3624             else:
3625                 try:
3626                     vm_list_xmlroot = XmlElementTree.fromstring(response.content)
3627                     for child in vm_list_xmlroot:
3628                         if child.tag.split("}")[1] == 'ProviderVdcReference':
3629                             provider_network = child.attrib.get('href')
3630                             # application/vnd.vmware.admin.providervdc+xml
3631                         if child.tag.split("}")[1] == 'Link':
3632                             if child.attrib.get('type') == 'application/vnd.vmware.vcloud.orgVdcNetwork+xml' \
3633                                     and child.attrib.get('rel') == 'add':
3634                                 add_vdc_rest_url = child.attrib.get('href')
3635                 except:
3636                     self.logger.debug("Failed parse respond for rest api call {}".format(vm_list_rest_call))
3637                     self.logger.debug("Respond body {}".format(response.content))
3638                     return None
3639
3640             # find  pvdc provided available network
3641             response = self.perform_request(req_type='GET',
3642                                             url=provider_network,
3643                                             headers=headers)
3644             if response.status_code != requests.codes.ok:
3645                 self.logger.debug("REST API call {} failed. Return status code {}".format(vm_list_rest_call,
3646                                                                                           response.status_code))
3647                 return None
3648
3649             if parent_network_uuid is None:
3650                 try:
3651                     vm_list_xmlroot = XmlElementTree.fromstring(response.content)
3652                     for child in vm_list_xmlroot.iter():
3653                         if child.tag.split("}")[1] == 'AvailableNetworks':
3654                             for networks in child.iter():
3655                                 # application/vnd.vmware.admin.network+xml
3656                                 if networks.attrib.get('href') is not None:
3657                                     available_networks = networks.attrib.get('href')
3658                                     break
3659                 except:
3660                     return None
3661
3662             try:
3663                 #Configure IP profile of the network
3664                 ip_profile = ip_profile if ip_profile is not None else DEFAULT_IP_PROFILE
3665
3666                 if 'subnet_address' not in ip_profile or ip_profile['subnet_address'] is None:
3667                     subnet_rand = random.randint(0, 255)
3668                     ip_base = "192.168.{}.".format(subnet_rand)
3669                     ip_profile['subnet_address'] = ip_base + "0/24"
3670                 else:
3671                     ip_base = ip_profile['subnet_address'].rsplit('.',1)[0] + '.'
3672
3673                 if 'gateway_address' not in ip_profile or ip_profile['gateway_address'] is None:
3674                     ip_profile['gateway_address']=ip_base + "1"
3675                 if 'dhcp_count' not in ip_profile or ip_profile['dhcp_count'] is None:
3676                     ip_profile['dhcp_count']=DEFAULT_IP_PROFILE['dhcp_count']
3677                 if 'dhcp_enabled' not in ip_profile or ip_profile['dhcp_enabled'] is None:
3678                     ip_profile['dhcp_enabled']=DEFAULT_IP_PROFILE['dhcp_enabled']
3679                 if 'dhcp_start_address' not in ip_profile or ip_profile['dhcp_start_address'] is None:
3680                     ip_profile['dhcp_start_address']=ip_base + "3"
3681                 if 'ip_version' not in ip_profile or ip_profile['ip_version'] is None:
3682                     ip_profile['ip_version']=DEFAULT_IP_PROFILE['ip_version']
3683                 if 'dns_address' not in ip_profile or ip_profile['dns_address'] is None:
3684                     ip_profile['dns_address']=ip_base + "2"
3685
3686                 gateway_address=ip_profile['gateway_address']
3687                 dhcp_count=int(ip_profile['dhcp_count'])
3688                 subnet_address=self.convert_cidr_to_netmask(ip_profile['subnet_address'])
3689
3690                 if ip_profile['dhcp_enabled']==True:
3691                     dhcp_enabled='true'
3692                 else:
3693                     dhcp_enabled='false'
3694                 dhcp_start_address=ip_profile['dhcp_start_address']
3695
3696                 #derive dhcp_end_address from dhcp_start_address & dhcp_count
3697                 end_ip_int = int(netaddr.IPAddress(dhcp_start_address))
3698                 end_ip_int += dhcp_count - 1
3699                 dhcp_end_address = str(netaddr.IPAddress(end_ip_int))
3700
3701                 ip_version=ip_profile['ip_version']
3702                 dns_address=ip_profile['dns_address']
3703             except KeyError as exp:
3704                 self.logger.debug("Create Network REST: Key error {}".format(exp))
3705                 raise vimconn.vimconnException("Create Network REST: Key error{}".format(exp))
3706
3707             # either use client provided UUID or search for a first available
3708             #  if both are not defined we return none
3709             if parent_network_uuid is not None:
3710                 url_list = [self.url, '/api/admin/network/', parent_network_uuid]
3711                 add_vdc_rest_url = ''.join(url_list)
3712
3713             #Creating all networks as Direct Org VDC type networks.
3714             #Unused in case of Underlay (data/ptp) network interface.
3715             fence_mode="bridged"
3716             is_inherited='false'
3717             dns_list = dns_address.split(";")
3718             dns1 = dns_list[0]
3719             dns2_text = ""
3720             if len(dns_list) >= 2:
3721                 dns2_text = "\n                                                <Dns2>{}</Dns2>\n".format(dns_list[1])
3722             data = """ <OrgVdcNetwork name="{0:s}" xmlns="http://www.vmware.com/vcloud/v1.5">
3723                             <Description>Openmano created</Description>
3724                                     <Configuration>
3725                                         <IpScopes>
3726                                             <IpScope>
3727                                                 <IsInherited>{1:s}</IsInherited>
3728                                                 <Gateway>{2:s}</Gateway>
3729                                                 <Netmask>{3:s}</Netmask>
3730                                                 <Dns1>{4:s}</Dns1>{5:s}
3731                                                 <IsEnabled>{6:s}</IsEnabled>
3732                                                 <IpRanges>
3733                                                     <IpRange>
3734                                                         <StartAddress>{7:s}</StartAddress>
3735                                                         <EndAddress>{8:s}</EndAddress>
3736                                                     </IpRange>
3737                                                 </IpRanges>
3738                                             </IpScope>
3739                                         </IpScopes>
3740                                         <ParentNetwork href="{9:s}"/>
3741                                         <FenceMode>{10:s}</FenceMode>
3742                                     </Configuration>
3743                                     <IsShared>{11:s}</IsShared>
3744                         </OrgVdcNetwork> """.format(escape(network_name), is_inherited, gateway_address,
3745                                                     subnet_address, dns1, dns2_text, dhcp_enabled,
3746                                                     dhcp_start_address, dhcp_end_address, available_networks,
3747                                                     fence_mode, isshared)
3748
3749             headers['Content-Type'] = 'application/vnd.vmware.vcloud.orgVdcNetwork+xml'
3750             try:
3751                 response = self.perform_request(req_type='POST',
3752                                            url=add_vdc_rest_url,
3753                                            headers=headers,
3754                                            data=data)
3755
3756                 if response.status_code != 201:
3757                     self.logger.debug("Create Network POST REST API call failed. Return status code {}, Response content: {}"
3758                                       .format(response.status_code,response.content))
3759                 else:
3760                     network_task = self.get_task_from_response(response.content)
3761                     self.logger.debug("Create Network REST : Waiting for Network creation complete")
3762                     time.sleep(5)
3763                     result = self.client.get_task_monitor().wait_for_success(task=network_task)
3764                     if result.get('status') == 'success':
3765                         return response.content
3766                     else:
3767                         self.logger.debug("create_network_rest task failed. Network Create response : {}"
3768                                           .format(response.content))
3769             except Exception as exp:
3770                 self.logger.debug("create_network_rest : Exception : {} ".format(exp))
3771
3772         return None
3773
3774     def convert_cidr_to_netmask(self, cidr_ip=None):
3775         """
3776         Method sets convert CIDR netmask address to normal IP format
3777         Args:
3778             cidr_ip : CIDR IP address
3779             Returns:
3780                 netmask : Converted netmask
3781         """
3782         if cidr_ip is not None:
3783             if '/' in cidr_ip:
3784                 network, net_bits = cidr_ip.split('/')
3785                 netmask = socket.inet_ntoa(struct.pack(">I", (0xffffffff << (32 - int(net_bits))) & 0xffffffff))
3786             else:
3787                 netmask = cidr_ip
3788             return netmask
3789         return None
3790
3791     def get_provider_rest(self, vca=None):
3792         """
3793         Method gets provider vdc view from vcloud director
3794
3795         Args:
3796             network_name - is network name to be created.
3797             parent_network_uuid - is parent provider vdc network that will be used for mapping.
3798             It optional attribute. by default if no parent network indicate the first available will be used.
3799
3800             Returns:
3801                 The return xml content of respond or None
3802         """
3803
3804         url_list = [self.url, '/api/admin']
3805         if vca:
3806             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
3807                        'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
3808             response = self.perform_request(req_type='GET',
3809                                             url=''.join(url_list),
3810                                             headers=headers)
3811
3812         if response.status_code == requests.codes.ok:
3813             return response.content
3814         return None
3815
3816     def create_vdc(self, vdc_name=None):
3817
3818         vdc_dict = {}
3819
3820         xml_content = self.create_vdc_from_tmpl_rest(vdc_name=vdc_name)
3821         if xml_content is not None:
3822             try:
3823                 task_resp_xmlroot = XmlElementTree.fromstring(xml_content)
3824                 for child in task_resp_xmlroot:
3825                     if child.tag.split("}")[1] == 'Owner':
3826                         vdc_id = child.attrib.get('href').split("/")[-1]
3827                         vdc_dict[vdc_id] = task_resp_xmlroot.get('href')
3828                         return vdc_dict
3829             except:
3830                 self.logger.debug("Respond body {}".format(xml_content))
3831
3832         return None
3833
3834     def create_vdc_from_tmpl_rest(self, vdc_name=None):
3835         """
3836         Method create vdc in vCloud director based on VDC template.
3837         it uses pre-defined template.
3838
3839         Args:
3840             vdc_name -  name of a new vdc.
3841
3842             Returns:
3843                 The return xml content of respond or None
3844         """
3845         # pre-requesite atleast one vdc template should be available in vCD
3846         self.logger.info("Creating new vdc {}".format(vdc_name))
3847         vca = self.connect_as_admin()
3848         if not vca:
3849             raise vimconn.vimconnConnectionException("Failed to connect vCD")
3850         if vdc_name is None:
3851             return None
3852
3853         url_list = [self.url, '/api/vdcTemplates']
3854         vm_list_rest_call = ''.join(url_list)
3855
3856         headers = {'Accept':'application/*+xml;version=' + API_VERSION,
3857                     'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
3858         response = self.perform_request(req_type='GET',
3859                                         url=vm_list_rest_call,
3860                                         headers=headers)
3861
3862         # container url to a template
3863         vdc_template_ref = None
3864         try:
3865             vm_list_xmlroot = XmlElementTree.fromstring(response.content)
3866             for child in vm_list_xmlroot:
3867                 # application/vnd.vmware.admin.providervdc+xml
3868                 # we need find a template from witch we instantiate VDC
3869                 if child.tag.split("}")[1] == 'VdcTemplate':
3870                     if child.attrib.get('type') == 'application/vnd.vmware.admin.vdcTemplate+xml':
3871                         vdc_template_ref = child.attrib.get('href')
3872         except:
3873             self.logger.debug("Failed parse respond for rest api call {}".format(vm_list_rest_call))
3874             self.logger.debug("Respond body {}".format(response.content))
3875             return None
3876
3877         # if we didn't found required pre defined template we return None
3878         if vdc_template_ref is None:
3879             return None
3880
3881         try:
3882             # instantiate vdc
3883             url_list = [self.url, '/api/org/', self.org_uuid, '/action/instantiate']
3884             vm_list_rest_call = ''.join(url_list)
3885             data = """<InstantiateVdcTemplateParams name="{0:s}" xmlns="http://www.vmware.com/vcloud/v1.5">
3886                                         <Source href="{1:s}"></Source>
3887                                         <Description>opnemano</Description>
3888                                         </InstantiateVdcTemplateParams>""".format(vdc_name, vdc_template_ref)
3889
3890             headers['Content-Type'] = 'application/vnd.vmware.vcloud.instantiateVdcTemplateParams+xml'
3891
3892             response = self.perform_request(req_type='POST',
3893                                             url=vm_list_rest_call,
3894                                             headers=headers,
3895                                             data=data)
3896
3897             vdc_task = self.get_task_from_response(response.content)
3898             self.client.get_task_monitor().wait_for_success(task=vdc_task)
3899
3900             # if we all ok we respond with content otherwise by default None
3901             if response.status_code >= 200 and response.status_code < 300:
3902                 return response.content
3903             return None
3904         except:
3905             self.logger.debug("Failed parse respond for rest api call {}".format(vm_list_rest_call))
3906             self.logger.debug("Respond body {}".format(response.content))
3907
3908         return None
3909
3910     def create_vdc_rest(self, vdc_name=None):
3911         """
3912         Method create network in vCloud director
3913
3914         Args:
3915             vdc_name - vdc name to be created
3916             Returns:
3917                 The return response
3918         """
3919
3920         self.logger.info("Creating new vdc {}".format(vdc_name))
3921
3922         vca = self.connect_as_admin()
3923         if not vca:
3924             raise vimconn.vimconnConnectionException("Failed to connect vCD")
3925         if vdc_name is None:
3926             return None
3927
3928         url_list = [self.url, '/api/admin/org/', self.org_uuid]
3929         vm_list_rest_call = ''.join(url_list)
3930
3931         if vca._session:
3932             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
3933                       'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
3934             response = self.perform_request(req_type='GET',
3935                                             url=vm_list_rest_call,
3936                                             headers=headers)
3937
3938             provider_vdc_ref = None
3939             add_vdc_rest_url = None
3940             available_networks = None
3941
3942             if response.status_code != requests.codes.ok:
3943                 self.logger.debug("REST API call {} failed. Return status code {}".format(vm_list_rest_call,
3944                                                                                           response.status_code))
3945                 return None
3946             else:
3947                 try:
3948                     vm_list_xmlroot = XmlElementTree.fromstring(response.content)
3949                     for child in vm_list_xmlroot:
3950                         # application/vnd.vmware.admin.providervdc+xml
3951                         if child.tag.split("}")[1] == 'Link':
3952                             if child.attrib.get('type') == 'application/vnd.vmware.admin.createVdcParams+xml' \
3953                                     and child.attrib.get('rel') == 'add':
3954                                 add_vdc_rest_url = child.attrib.get('href')
3955                 except:
3956                     self.logger.debug("Failed parse respond for rest api call {}".format(vm_list_rest_call))
3957                     self.logger.debug("Respond body {}".format(response.content))
3958                     return None
3959
3960                 response = self.get_provider_rest(vca=vca)
3961                 try:
3962                     vm_list_xmlroot = XmlElementTree.fromstring(response)
3963                     for child in vm_list_xmlroot:
3964                         if child.tag.split("}")[1] == 'ProviderVdcReferences':
3965                             for sub_child in child:
3966                                 provider_vdc_ref = sub_child.attrib.get('href')
3967                 except:
3968                     self.logger.debug("Failed parse respond for rest api call {}".format(vm_list_rest_call))
3969                     self.logger.debug("Respond body {}".format(response))
3970                     return None
3971
3972                 if add_vdc_rest_url is not None and provider_vdc_ref is not None:
3973                     data = """ <CreateVdcParams name="{0:s}" xmlns="http://www.vmware.com/vcloud/v1.5"><Description>{1:s}</Description>
3974                             <AllocationModel>ReservationPool</AllocationModel>
3975                             <ComputeCapacity><Cpu><Units>MHz</Units><Allocated>2048</Allocated><Limit>2048</Limit></Cpu>
3976                             <Memory><Units>MB</Units><Allocated>2048</Allocated><Limit>2048</Limit></Memory>
3977                             </ComputeCapacity><NicQuota>0</NicQuota><NetworkQuota>100</NetworkQuota>
3978                             <VdcStorageProfile><Enabled>true</Enabled><Units>MB</Units><Limit>20480</Limit><Default>true</Default></VdcStorageProfile>
3979                             <ProviderVdcReference
3980                             name="Main Provider"
3981                             href="{2:s}" />
3982                     <UsesFastProvisioning>true</UsesFastProvisioning></CreateVdcParams>""".format(escape(vdc_name),
3983                                                                                                   escape(vdc_name),
3984                                                                                                   provider_vdc_ref)
3985
3986                     headers['Content-Type'] = 'application/vnd.vmware.admin.createVdcParams+xml'
3987
3988                     response = self.perform_request(req_type='POST',
3989                                                     url=add_vdc_rest_url,
3990                                                     headers=headers,
3991                                                     data=data)
3992
3993                     # if we all ok we respond with content otherwise by default None
3994                     if response.status_code == 201:
3995                         return response.content
3996         return None
3997
3998     def get_vapp_details_rest(self, vapp_uuid=None, need_admin_access=False):
3999         """
4000         Method retrieve vapp detail from vCloud director
4001
4002         Args:
4003             vapp_uuid - is vapp identifier.
4004
4005             Returns:
4006                 The return network uuid or return None
4007         """
4008
4009         parsed_respond = {}
4010         vca = None
4011
4012         if need_admin_access:
4013             vca = self.connect_as_admin()
4014         else:
4015             vca = self.client
4016
4017         if not vca:
4018             raise vimconn.vimconnConnectionException("Failed to connect vCD")
4019         if vapp_uuid is None:
4020             return None
4021
4022         url_list = [self.url, '/api/vApp/vapp-', vapp_uuid]
4023         get_vapp_restcall = ''.join(url_list)
4024
4025         if vca._session:
4026             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
4027                        'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
4028             response = self.perform_request(req_type='GET',
4029                                             url=get_vapp_restcall,
4030                                             headers=headers)
4031
4032             if response.status_code == 403:
4033                 if need_admin_access == False:
4034                     response = self.retry_rest('GET', get_vapp_restcall)
4035
4036             if response.status_code != requests.codes.ok:
4037                 self.logger.debug("REST API call {} failed. Return status code {}".format(get_vapp_restcall,
4038                                                                                           response.status_code))
4039                 return parsed_respond
4040
4041             try:
4042                 xmlroot_respond = XmlElementTree.fromstring(response.content)
4043                 parsed_respond['ovfDescriptorUploaded'] = xmlroot_respond.attrib['ovfDescriptorUploaded']
4044
4045                 namespaces = {"vssd":"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData" ,
4046                               'ovf': 'http://schemas.dmtf.org/ovf/envelope/1',
4047                               'vmw': 'http://www.vmware.com/schema/ovf',
4048                               'vm': 'http://www.vmware.com/vcloud/v1.5',
4049                               'rasd':"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData",
4050                               "vmext":"http://www.vmware.com/vcloud/extension/v1.5",
4051                               "xmlns":"http://www.vmware.com/vcloud/v1.5"
4052                              }
4053
4054                 created_section = xmlroot_respond.find('vm:DateCreated', namespaces)
4055                 if created_section is not None:
4056                     parsed_respond['created'] = created_section.text
4057
4058                 network_section = xmlroot_respond.find('vm:NetworkConfigSection/vm:NetworkConfig', namespaces)
4059                 if network_section is not None and 'networkName' in network_section.attrib:
4060                     parsed_respond['networkname'] = network_section.attrib['networkName']
4061
4062                 ipscopes_section = \
4063                     xmlroot_respond.find('vm:NetworkConfigSection/vm:NetworkConfig/vm:Configuration/vm:IpScopes',
4064                                          namespaces)
4065                 if ipscopes_section is not None:
4066                     for ipscope in ipscopes_section:
4067                         for scope in ipscope:
4068                             tag_key = scope.tag.split("}")[1]
4069                             if tag_key == 'IpRanges':
4070                                 ip_ranges = scope.getchildren()
4071                                 for ipblock in ip_ranges:
4072                                     for block in ipblock:
4073                                         parsed_respond[block.tag.split("}")[1]] = block.text
4074                             else:
4075                                 parsed_respond[tag_key] = scope.text
4076
4077                 # parse children section for other attrib
4078                 children_section = xmlroot_respond.find('vm:Children/', namespaces)
4079                 if children_section is not None:
4080                     parsed_respond['name'] = children_section.attrib['name']
4081                     parsed_respond['nestedHypervisorEnabled'] = children_section.attrib['nestedHypervisorEnabled'] \
4082                      if  "nestedHypervisorEnabled" in children_section.attrib else None
4083                     parsed_respond['deployed'] = children_section.attrib['deployed']
4084                     parsed_respond['status'] = children_section.attrib['status']
4085                     parsed_respond['vmuuid'] = children_section.attrib['id'].split(":")[-1]
4086                     network_adapter = children_section.find('vm:NetworkConnectionSection', namespaces)
4087                     nic_list = []
4088                     for adapters in network_adapter:
4089                         adapter_key = adapters.tag.split("}")[1]
4090                         if adapter_key == 'PrimaryNetworkConnectionIndex':
4091                             parsed_respond['primarynetwork'] = adapters.text
4092                         if adapter_key == 'NetworkConnection':
4093                             vnic = {}
4094                             if 'network' in adapters.attrib:
4095                                 vnic['network'] = adapters.attrib['network']
4096                             for adapter in adapters:
4097                                 setting_key = adapter.tag.split("}")[1]
4098                                 vnic[setting_key] = adapter.text
4099                             nic_list.append(vnic)
4100
4101                     for link in children_section:
4102                         if link.tag.split("}")[1] == 'Link' and 'rel' in link.attrib:
4103                             if link.attrib['rel'] == 'screen:acquireTicket':
4104                                 parsed_respond['acquireTicket'] = link.attrib
4105                             if link.attrib['rel'] == 'screen:acquireMksTicket':
4106                                 parsed_respond['acquireMksTicket'] = link.attrib
4107
4108                     parsed_respond['interfaces'] = nic_list
4109                     vCloud_extension_section = children_section.find('xmlns:VCloudExtension', namespaces)
4110                     if vCloud_extension_section is not None:
4111                         vm_vcenter_info = {}
4112                         vim_info = vCloud_extension_section.find('vmext:VmVimInfo', namespaces)
4113                         vmext = vim_info.find('vmext:VmVimObjectRef', namespaces)
4114                         if vmext is not None:
4115                             vm_vcenter_info["vm_moref_id"] = vmext.find('vmext:MoRef', namespaces).text
4116                         parsed_respond["vm_vcenter_info"]= vm_vcenter_info
4117
4118                     virtual_hardware_section = children_section.find('ovf:VirtualHardwareSection', namespaces)
4119                     vm_virtual_hardware_info = {}
4120                     if virtual_hardware_section is not None:
4121                         for item in virtual_hardware_section.iterfind('ovf:Item',namespaces):
4122                             if item.find("rasd:Description",namespaces).text == "Hard disk":
4123                                 disk_size = item.find("rasd:HostResource" ,namespaces
4124                                                 ).attrib["{"+namespaces['vm']+"}capacity"]
4125
4126                                 vm_virtual_hardware_info["disk_size"]= disk_size
4127                                 break
4128
4129                         for link in virtual_hardware_section:
4130                             if link.tag.split("}")[1] == 'Link' and 'rel' in link.attrib:
4131                                 if link.attrib['rel'] == 'edit' and link.attrib['href'].endswith("/disks"):
4132                                     vm_virtual_hardware_info["disk_edit_href"] = link.attrib['href']
4133                                     break
4134
4135                     parsed_respond["vm_virtual_hardware"]= vm_virtual_hardware_info
4136             except Exception as exp :
4137                 self.logger.info("Error occurred calling rest api for getting vApp details {}".format(exp))
4138         return parsed_respond
4139
4140     def acquire_console(self, vm_uuid=None):
4141
4142         if vm_uuid is None:
4143             return None
4144         if self.client._session:
4145             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
4146                        'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
4147             vm_dict = self.get_vapp_details_rest(vapp_uuid=vm_uuid)
4148             console_dict = vm_dict['acquireTicket']
4149             console_rest_call = console_dict['href']
4150
4151             response = self.perform_request(req_type='POST',
4152                                             url=console_rest_call,
4153                                             headers=headers)
4154
4155             if response.status_code == 403:
4156                 response = self.retry_rest('POST', console_rest_call)
4157
4158             if response.status_code == requests.codes.ok:
4159                 return response.content
4160
4161         return None
4162
4163     def modify_vm_disk(self, vapp_uuid, flavor_disk):
4164         """
4165         Method retrieve vm disk details
4166
4167         Args:
4168             vapp_uuid - is vapp identifier.
4169             flavor_disk - disk size as specified in VNFD (flavor)
4170
4171             Returns:
4172                 The return network uuid or return None
4173         """
4174         status = None
4175         try:
4176             #Flavor disk is in GB convert it into MB
4177             flavor_disk = int(flavor_disk) * 1024
4178             vm_details = self.get_vapp_details_rest(vapp_uuid)
4179             if vm_details:
4180                 vm_name = vm_details["name"]
4181                 self.logger.info("VM: {} flavor_disk :{}".format(vm_name , flavor_disk))
4182
4183             if vm_details and "vm_virtual_hardware" in vm_details:
4184                 vm_disk = int(vm_details["vm_virtual_hardware"]["disk_size"])
4185                 disk_edit_href = vm_details["vm_virtual_hardware"]["disk_edit_href"]
4186
4187                 self.logger.info("VM: {} VM_disk :{}".format(vm_name , vm_disk))
4188
4189                 if flavor_disk > vm_disk:
4190                     status = self.modify_vm_disk_rest(disk_edit_href ,flavor_disk)
4191                     self.logger.info("Modify disk of VM {} from {} to {} MB".format(vm_name,
4192                                                          vm_disk,  flavor_disk ))
4193                 else:
4194                     status = True
4195                     self.logger.info("No need to modify disk of VM {}".format(vm_name))
4196
4197             return status
4198         except Exception as exp:
4199             self.logger.info("Error occurred while modifing disk size {}".format(exp))
4200
4201
4202     def modify_vm_disk_rest(self, disk_href , disk_size):
4203         """
4204         Method retrieve modify vm disk size
4205
4206         Args:
4207             disk_href - vCD API URL to GET and PUT disk data
4208             disk_size - disk size as specified in VNFD (flavor)
4209
4210             Returns:
4211                 The return network uuid or return None
4212         """
4213         if disk_href is None or disk_size is None:
4214             return None
4215
4216         if self.client._session:
4217                 headers = {'Accept':'application/*+xml;version=' + API_VERSION,
4218                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
4219                 response = self.perform_request(req_type='GET',
4220                                                 url=disk_href,
4221                                                 headers=headers)
4222
4223         if response.status_code == 403:
4224             response = self.retry_rest('GET', disk_href)
4225
4226         if response.status_code != requests.codes.ok:
4227             self.logger.debug("GET REST API call {} failed. Return status code {}".format(disk_href,
4228                                                                             response.status_code))
4229             return None
4230         try:
4231             lxmlroot_respond = lxmlElementTree.fromstring(response.content)
4232             namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.iteritems() if prefix}
4233             #For python3
4234             #namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.items() if prefix}
4235             namespaces["xmlns"]= "http://www.vmware.com/vcloud/v1.5"
4236
4237             for item in lxmlroot_respond.iterfind('xmlns:Item',namespaces):
4238                 if item.find("rasd:Description",namespaces).text == "Hard disk":
4239                     disk_item = item.find("rasd:HostResource" ,namespaces )
4240                     if disk_item is not None:
4241                         disk_item.attrib["{"+namespaces['xmlns']+"}capacity"] = str(disk_size)
4242                         break
4243
4244             data = lxmlElementTree.tostring(lxmlroot_respond, encoding='utf8', method='xml',
4245                                              xml_declaration=True)
4246
4247             #Send PUT request to modify disk size
4248             headers['Content-Type'] = 'application/vnd.vmware.vcloud.rasdItemsList+xml; charset=ISO-8859-1'
4249
4250             response = self.perform_request(req_type='PUT',
4251                                                 url=disk_href,
4252                                                 headers=headers,
4253                                                 data=data)
4254             if response.status_code == 403:
4255                 add_headers = {'Content-Type': headers['Content-Type']}
4256                 response = self.retry_rest('PUT', disk_href, add_headers, data)
4257
4258             if response.status_code != 202:
4259                 self.logger.debug("PUT REST API call {} failed. Return status code {}".format(disk_href,
4260                                                                             response.status_code))
4261             else:
4262                 modify_disk_task = self.get_task_from_response(response.content)
4263                 result = self.client.get_task_monitor().wait_for_success(task=modify_disk_task)
4264                 if result.get('status') == 'success':
4265                     return True
4266                 else:
4267                     return False
4268             return None
4269
4270         except Exception as exp :
4271                 self.logger.info("Error occurred calling rest api for modifing disk size {}".format(exp))
4272                 return None
4273
4274     def add_pci_devices(self, vapp_uuid , pci_devices , vmname_andid):
4275         """
4276             Method to attach pci devices to VM
4277
4278              Args:
4279                 vapp_uuid - uuid of vApp/VM
4280                 pci_devices - pci devices infromation as specified in VNFD (flavor)
4281
4282             Returns:
4283                 The status of add pci device task , vm object and
4284                 vcenter_conect object
4285         """
4286         vm_obj = None
4287         self.logger.info("Add pci devices {} into vApp {}".format(pci_devices , vapp_uuid))
4288         vcenter_conect, content = self.get_vcenter_content()
4289         vm_moref_id = self.get_vm_moref_id(vapp_uuid)
4290
4291         if vm_moref_id:
4292             try:
4293                 no_of_pci_devices = len(pci_devices)
4294                 if no_of_pci_devices > 0:
4295                     #Get VM and its host
4296                     host_obj, vm_obj = self.get_vm_obj(content, vm_moref_id)
4297                     self.logger.info("VM {} is currently on host {}".format(vm_obj, host_obj))
4298                     if host_obj and vm_obj:
4299                         #get PCI devies from host on which vapp is currently installed
4300                         avilable_pci_devices = self.get_pci_devices(host_obj, no_of_pci_devices)
4301
4302                         if avilable_pci_devices is None:
4303                             #find other hosts with active pci devices
4304                             new_host_obj , avilable_pci_devices = self.get_host_and_PCIdevices(
4305                                                                 content,
4306                                                                 no_of_pci_devices
4307                                                                 )
4308
4309                             if new_host_obj is not None and avilable_pci_devices is not None and len(avilable_pci_devices)> 0:
4310                                 #Migrate vm to the host where PCI devices are availble
4311                                 self.logger.info("Relocate VM {} on new host {}".format(vm_obj, new_host_obj))
4312                                 task = self.relocate_vm(new_host_obj, vm_obj)
4313                                 if task is not None:
4314                                     result = self.wait_for_vcenter_task(task, vcenter_conect)
4315                                     self.logger.info("Migrate VM status: {}".format(result))
4316                                     host_obj = new_host_obj
4317                                 else:
4318                                     self.logger.info("Fail to migrate VM : {}".format(result))
4319                                     raise vimconn.vimconnNotFoundException(
4320                                     "Fail to migrate VM : {} to host {}".format(
4321                                                     vmname_andid,
4322                                                     new_host_obj)
4323                                         )
4324
4325                         if host_obj is not None and avilable_pci_devices is not None and len(avilable_pci_devices)> 0:
4326                             #Add PCI devices one by one
4327                             for pci_device in avilable_pci_devices:
4328                                 task = self.add_pci_to_vm(host_obj, vm_obj, pci_device)
4329                                 if task:
4330                                     status= self.wait_for_vcenter_task(task, vcenter_conect)
4331                                     if status:
4332                                         self.logger.info("Added PCI device {} to VM {}".format(pci_device,str(vm_obj)))
4333                                 else:
4334                                     self.logger.error("Fail to add PCI device {} to VM {}".format(pci_device,str(vm_obj)))
4335                             return True, vm_obj, vcenter_conect
4336                         else:
4337                             self.logger.error("Currently there is no host with"\
4338                                               " {} number of avaialble PCI devices required for VM {}".format(
4339                                                                             no_of_pci_devices,
4340                                                                             vmname_andid)
4341                                               )
4342                             raise vimconn.vimconnNotFoundException(
4343                                     "Currently there is no host with {} "\
4344                                     "number of avaialble PCI devices required for VM {}".format(
4345                                                                             no_of_pci_devices,
4346                                                                             vmname_andid))
4347                 else:
4348                     self.logger.debug("No infromation about PCI devices {} ",pci_devices)
4349
4350             except vmodl.MethodFault as error:
4351                 self.logger.error("Error occurred while adding PCI devices {} ",error)
4352         return None, vm_obj, vcenter_conect
4353
4354     def get_vm_obj(self, content, mob_id):
4355         """
4356             Method to get the vsphere VM object associated with a given morf ID
4357              Args:
4358                 vapp_uuid - uuid of vApp/VM
4359                 content - vCenter content object
4360                 mob_id - mob_id of VM
4361
4362             Returns:
4363                     VM and host object
4364         """
4365         vm_obj = None
4366         host_obj = None
4367         try :
4368             container = content.viewManager.CreateContainerView(content.rootFolder,
4369                                                         [vim.VirtualMachine], True
4370                                                         )
4371             for vm in container.view:
4372                 mobID = vm._GetMoId()
4373                 if mobID == mob_id:
4374                     vm_obj = vm
4375                     host_obj = vm_obj.runtime.host
4376                     break
4377         except Exception as exp:
4378             self.logger.error("Error occurred while finding VM object : {}".format(exp))
4379         return host_obj, vm_obj
4380
4381     def get_pci_devices(self, host, need_devices):
4382         """
4383             Method to get the details of pci devices on given host
4384              Args:
4385                 host - vSphere host object
4386                 need_devices - number of pci devices needed on host
4387
4388              Returns:
4389                 array of pci devices
4390         """
4391         all_devices = []
4392         all_device_ids = []
4393         used_devices_ids = []
4394
4395         try:
4396             if host:
4397                 pciPassthruInfo = host.config.pciPassthruInfo
4398                 pciDevies = host.hardware.pciDevice
4399
4400             for pci_status in pciPassthruInfo:
4401                 if pci_status.passthruActive:
4402                     for device in pciDevies:
4403                         if device.id == pci_status.id:
4404                             all_device_ids.append(device.id)
4405                             all_devices.append(device)
4406
4407             #check if devices are in use
4408             avalible_devices = all_devices
4409             for vm in host.vm:
4410                 if vm.runtime.powerState == vim.VirtualMachinePowerState.poweredOn:
4411                     vm_devices = vm.config.hardware.device
4412                     for device in vm_devices:
4413                         if type(device) is vim.vm.device.VirtualPCIPassthrough:
4414                             if device.backing.id in all_device_ids:
4415                                 for use_device in avalible_devices:
4416                                     if use_device.id == device.backing.id:
4417                                         avalible_devices.remove(use_device)
4418                                 used_devices_ids.append(device.backing.id)
4419                                 self.logger.debug("Device {} from devices {}"\
4420                                         "is in use".format(device.backing.id,
4421                                                            device)
4422                                             )
4423             if len(avalible_devices) < need_devices:
4424                 self.logger.debug("Host {} don't have {} number of active devices".format(host,
4425                                                                             need_devices))
4426                 self.logger.debug("found only {} devives {}".format(len(avalible_devices),
4427                                                                     avalible_devices))
4428                 return None
4429             else:
4430                 required_devices = avalible_devices[:need_devices]
4431                 self.logger.info("Found {} PCI devivces on host {} but required only {}".format(
4432                                                             len(avalible_devices),
4433                                                             host,
4434                                                             need_devices))
4435                 self.logger.info("Retruning {} devices as {}".format(need_devices,
4436                                                                 required_devices ))
4437                 return required_devices
4438
4439         except Exception as exp:
4440             self.logger.error("Error {} occurred while finding pci devices on host: {}".format(exp, host))
4441
4442         return None
4443
4444     def get_host_and_PCIdevices(self, content, need_devices):
4445         """
4446          Method to get the details of pci devices infromation on all hosts
4447
4448             Args:
4449                 content - vSphere host object
4450                 need_devices - number of pci devices needed on host
4451
4452             Returns:
4453                  array of pci devices and host object
4454         """
4455         host_obj = None
4456         pci_device_objs = None
4457         try:
4458             if content:
4459                 container = content.viewManager.CreateContainerView(content.rootFolder,
4460                                                             [vim.HostSystem], True)
4461                 for host in container.view:
4462                     devices = self.get_pci_devices(host, need_devices)
4463                     if devices:
4464                         host_obj = host
4465                         pci_device_objs = devices
4466                         break
4467         except Exception as exp:
4468             self.logger.error("Error {} occurred while finding pci devices on host: {}".format(exp, host_obj))
4469
4470         return host_obj,pci_device_objs
4471
4472     def relocate_vm(self, dest_host, vm) :
4473         """
4474          Method to get the relocate VM to new host
4475
4476             Args:
4477                 dest_host - vSphere host object
4478                 vm - vSphere VM object
4479
4480             Returns:
4481                 task object
4482         """
4483         task = None
4484         try:
4485             relocate_spec = vim.vm.RelocateSpec(host=dest_host)
4486             task = vm.Relocate(relocate_spec)
4487             self.logger.info("Migrating {} to destination host {}".format(vm, dest_host))
4488         except Exception as exp:
4489             self.logger.error("Error occurred while relocate VM {} to new host {}: {}".format(
4490                                                                             dest_host, vm, exp))
4491         return task
4492
4493     def wait_for_vcenter_task(self, task, actionName='job', hideResult=False):
4494         """
4495         Waits and provides updates on a vSphere task
4496         """
4497         while task.info.state == vim.TaskInfo.State.running:
4498             time.sleep(2)
4499
4500         if task.info.state == vim.TaskInfo.State.success:
4501             if task.info.result is not None and not hideResult:
4502                 self.logger.info('{} completed successfully, result: {}'.format(
4503                                                             actionName,
4504                                                             task.info.result))
4505             else:
4506                 self.logger.info('Task {} completed successfully.'.format(actionName))
4507         else:
4508             self.logger.error('{} did not complete successfully: {} '.format(
4509                                                             actionName,
4510                                                             task.info.error)
4511                               )
4512
4513         return task.info.result
4514
4515     def add_pci_to_vm(self,host_object, vm_object, host_pci_dev):
4516         """
4517          Method to add pci device in given VM
4518
4519             Args:
4520                 host_object - vSphere host object
4521                 vm_object - vSphere VM object
4522                 host_pci_dev -  host_pci_dev must be one of the devices from the
4523                                 host_object.hardware.pciDevice list
4524                                 which is configured as a PCI passthrough device
4525
4526             Returns:
4527                 task object
4528         """
4529         task = None
4530         if vm_object and host_object and host_pci_dev:
4531             try :
4532                 #Add PCI device to VM
4533                 pci_passthroughs = vm_object.environmentBrowser.QueryConfigTarget(host=None).pciPassthrough
4534                 systemid_by_pciid = {item.pciDevice.id: item.systemId for item in pci_passthroughs}
4535
4536                 if host_pci_dev.id not in systemid_by_pciid:
4537                     self.logger.error("Device {} is not a passthrough device ".format(host_pci_dev))
4538                     return None
4539
4540                 deviceId = hex(host_pci_dev.deviceId % 2**16).lstrip('0x')
4541                 backing = vim.VirtualPCIPassthroughDeviceBackingInfo(deviceId=deviceId,
4542                                             id=host_pci_dev.id,
4543                                             systemId=systemid_by_pciid[host_pci_dev.id],
4544                                             vendorId=host_pci_dev.vendorId,
4545                                             deviceName=host_pci_dev.deviceName)
4546
4547                 hba_object = vim.VirtualPCIPassthrough(key=-100, backing=backing)
4548
4549                 new_device_config = vim.VirtualDeviceConfigSpec(device=hba_object)
4550                 new_device_config.operation = "add"
4551                 vmConfigSpec = vim.vm.ConfigSpec()
4552                 vmConfigSpec.deviceChange = [new_device_config]
4553
4554                 task = vm_object.ReconfigVM_Task(spec=vmConfigSpec)
4555                 self.logger.info("Adding PCI device {} into VM {} from host {} ".format(
4556                                                             host_pci_dev, vm_object, host_object)
4557                                 )
4558             except Exception as exp:
4559                 self.logger.error("Error occurred while adding pci devive {} to VM {}: {}".format(
4560                                                                             host_pci_dev,
4561                                                                             vm_object,
4562                                                                              exp))
4563         return task
4564
4565     def get_vm_vcenter_info(self):
4566         """
4567         Method to get details of vCenter and vm
4568
4569             Args:
4570                 vapp_uuid - uuid of vApp or VM
4571
4572             Returns:
4573                 Moref Id of VM and deails of vCenter
4574         """
4575         vm_vcenter_info = {}
4576
4577         if self.vcenter_ip is not None:
4578             vm_vcenter_info["vm_vcenter_ip"] = self.vcenter_ip
4579         else:
4580             raise vimconn.vimconnException(message="vCenter IP is not provided."\
4581                                            " Please provide vCenter IP while attaching datacenter to tenant in --config")
4582         if self.vcenter_port is not None:
4583             vm_vcenter_info["vm_vcenter_port"] = self.vcenter_port
4584         else:
4585             raise vimconn.vimconnException(message="vCenter port is not provided."\
4586                                            " Please provide vCenter port while attaching datacenter to tenant in --config")
4587         if self.vcenter_user is not None:
4588             vm_vcenter_info["vm_vcenter_user"] = self.vcenter_user
4589         else:
4590             raise vimconn.vimconnException(message="vCenter user is not provided."\
4591                                            " Please provide vCenter user while attaching datacenter to tenant in --config")
4592
4593         if self.vcenter_password is not None:
4594             vm_vcenter_info["vm_vcenter_password"] = self.vcenter_password
4595         else:
4596             raise vimconn.vimconnException(message="vCenter user password is not provided."\
4597                                            " Please provide vCenter user password while attaching datacenter to tenant in --config")
4598
4599         return vm_vcenter_info
4600
4601
4602     def get_vm_pci_details(self, vmuuid):
4603         """
4604             Method to get VM PCI device details from vCenter
4605
4606             Args:
4607                 vm_obj - vSphere VM object
4608
4609             Returns:
4610                 dict of PCI devives attached to VM
4611
4612         """
4613         vm_pci_devices_info = {}
4614         try:
4615             vcenter_conect, content = self.get_vcenter_content()
4616             vm_moref_id = self.get_vm_moref_id(vmuuid)
4617             if vm_moref_id:
4618                 #Get VM and its host
4619                 if content:
4620                     host_obj, vm_obj = self.get_vm_obj(content, vm_moref_id)
4621                     if host_obj and vm_obj:
4622                         vm_pci_devices_info["host_name"]= host_obj.name
4623                         vm_pci_devices_info["host_ip"]= host_obj.config.network.vnic[0].spec.ip.ipAddress
4624                         for device in vm_obj.config.hardware.device:
4625                             if type(device) == vim.vm.device.VirtualPCIPassthrough:
4626                                 device_details={'devide_id':device.backing.id,
4627                                                 'pciSlotNumber':device.slotInfo.pciSlotNumber,
4628                                             }
4629                                 vm_pci_devices_info[device.deviceInfo.label] = device_details
4630                 else:
4631                     self.logger.error("Can not connect to vCenter while getting "\
4632                                           "PCI devices infromationn")
4633                 return vm_pci_devices_info
4634         except Exception as exp:
4635             self.logger.error("Error occurred while getting VM infromationn"\
4636                              " for VM : {}".format(exp))
4637             raise vimconn.vimconnException(message=exp)
4638
4639     def add_network_adapter_to_vms(self, vapp, network_name, primary_nic_index, nicIndex, net, nic_type=None):
4640         """
4641             Method to add network adapter type to vm
4642             Args :
4643                 network_name - name of network
4644                 primary_nic_index - int value for primary nic index
4645                 nicIndex - int value for nic index
4646                 nic_type - specify model name to which add to vm
4647             Returns:
4648                 None
4649         """
4650
4651         self.logger.info("Add network adapter to VM: network_name {} nicIndex {}".\
4652                          format(network_name, nicIndex))
4653         try:
4654             ip_address = None
4655             floating_ip = False
4656             mac_address = None
4657             if 'floating_ip' in net: floating_ip = net['floating_ip']
4658
4659             # Stub for ip_address feature
4660             if 'ip_address' in net: ip_address = net['ip_address']
4661
4662             if 'mac_address' in net: mac_address = net['mac_address']
4663
4664             if floating_ip:
4665                 allocation_mode = "POOL"
4666             elif ip_address:
4667                 allocation_mode = "MANUAL"
4668             else:
4669                 allocation_mode = "DHCP"
4670
4671             if not nic_type:
4672                 for vms in vapp.get_all_vms():
4673                     vm_id = vms.get('id').split(':')[-1]
4674
4675                     url_rest_call = "{}/api/vApp/vm-{}/networkConnectionSection/".format(self.url, vm_id)
4676
4677                     headers = {'Accept':'application/*+xml;version=' + API_VERSION,
4678                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
4679                     response = self.perform_request(req_type='GET',
4680                                                     url=url_rest_call,
4681                                                     headers=headers)
4682
4683                     if response.status_code == 403:
4684                         response = self.retry_rest('GET', url_rest_call)
4685
4686                     if response.status_code != 200:
4687                         self.logger.error("REST call {} failed reason : {}"\
4688                                              "status code : {}".format(url_rest_call,
4689                                                                     response.content,
4690                                                                response.status_code))
4691                         raise vimconn.vimconnException("add_network_adapter_to_vms : Failed to get "\
4692                                                                          "network connection section")
4693
4694                     data = response.content
4695                     data = data.split('<Link rel="edit"')[0]
4696                     if '<PrimaryNetworkConnectionIndex>' not in data:
4697                         item = """<PrimaryNetworkConnectionIndex>{}</PrimaryNetworkConnectionIndex>
4698                                 <NetworkConnection network="{}">
4699                                 <NetworkConnectionIndex>{}</NetworkConnectionIndex>
4700                                 <IsConnected>true</IsConnected>
4701                                 <IpAddressAllocationMode>{}</IpAddressAllocationMode>
4702                                 </NetworkConnection>""".format(primary_nic_index, network_name, nicIndex,
4703                                                                                          allocation_mode)
4704                         # Stub for ip_address feature
4705                         if ip_address:
4706                             ip_tag = '<IpAddress>{}</IpAddress>'.format(ip_address)
4707                             item =  item.replace('</NetworkConnectionIndex>\n','</NetworkConnectionIndex>\n{}\n'.format(ip_tag))
4708
4709                         if mac_address:
4710                             mac_tag = '<MACAddress>{}</MACAddress>'.format(mac_address)
4711                             item =  item.replace('</IsConnected>\n','</IsConnected>\n{}\n'.format(mac_tag))
4712
4713                         data = data.replace('</ovf:Info>\n','</ovf:Info>\n{}\n</NetworkConnectionSection>'.format(item))
4714                     else:
4715                         new_item = """<NetworkConnection network="{}">
4716                                     <NetworkConnectionIndex>{}</NetworkConnectionIndex>
4717                                     <IsConnected>true</IsConnected>
4718                                     <IpAddressAllocationMode>{}</IpAddressAllocationMode>
4719                                     </NetworkConnection>""".format(network_name, nicIndex,
4720                                                                           allocation_mode)
4721                         # Stub for ip_address feature
4722                         if ip_address:
4723                             ip_tag = '<IpAddress>{}</IpAddress>'.format(ip_address)
4724                             new_item =  new_item.replace('</NetworkConnectionIndex>\n','</NetworkConnectionIndex>\n{}\n'.format(ip_tag))
4725
4726                         if mac_address:
4727                             mac_tag = '<MACAddress>{}</MACAddress>'.format(mac_address)
4728                             new_item =  new_item.replace('</IsConnected>\n','</IsConnected>\n{}\n'.format(mac_tag))
4729
4730                         data = data + new_item + '</NetworkConnectionSection>'
4731
4732                     headers['Content-Type'] = 'application/vnd.vmware.vcloud.networkConnectionSection+xml'
4733
4734                     response = self.perform_request(req_type='PUT',
4735                                                     url=url_rest_call,
4736                                                     headers=headers,
4737                                                     data=data)
4738
4739                     if response.status_code == 403:
4740                         add_headers = {'Content-Type': headers['Content-Type']}
4741                         response = self.retry_rest('PUT', url_rest_call, add_headers, data)
4742
4743                     if response.status_code != 202:
4744                         self.logger.error("REST call {} failed reason : {}"\
4745                                             "status code : {} ".format(url_rest_call,
4746                                                                     response.content,
4747                                                                response.status_code))
4748                         raise vimconn.vimconnException("add_network_adapter_to_vms : Failed to update "\
4749                                                                             "network connection section")
4750                     else:
4751                         nic_task = self.get_task_from_response(response.content)
4752                         result = self.client.get_task_monitor().wait_for_success(task=nic_task)
4753                         if result.get('status') == 'success':
4754                             self.logger.info("add_network_adapter_to_vms(): VM {} conneced to "\
4755                                                                "default NIC type".format(vm_id))
4756                         else:
4757                             self.logger.error("add_network_adapter_to_vms(): VM {} failed to "\
4758                                                               "connect NIC type".format(vm_id))
4759             else:
4760                 for vms in vapp.get_all_vms():
4761                     vm_id = vms.get('id').split(':')[-1]
4762
4763
4764                     url_rest_call = "{}/api/vApp/vm-{}/networkConnectionSection/".format(self.url, vm_id)
4765
4766                     headers = {'Accept':'application/*+xml;version=' + API_VERSION,
4767                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
4768                     response = self.perform_request(req_type='GET',
4769                                                     url=url_rest_call,
4770                                                     headers=headers)
4771
4772                     if response.status_code == 403:
4773                         response = self.retry_rest('GET', url_rest_call)
4774
4775                     if response.status_code != 200:
4776                         self.logger.error("REST call {} failed reason : {}"\
4777                                             "status code : {}".format(url_rest_call,
4778                                                                    response.content,
4779                                                               response.status_code))
4780                         raise vimconn.vimconnException("add_network_adapter_to_vms : Failed to get "\
4781                                                                         "network connection section")
4782                     data = response.content
4783                     data = data.split('<Link rel="edit"')[0]
4784                     if '<PrimaryNetworkConnectionIndex>' not in data:
4785                         item = """<PrimaryNetworkConnectionIndex>{}</PrimaryNetworkConnectionIndex>
4786                                 <NetworkConnection network="{}">
4787                                 <NetworkConnectionIndex>{}</NetworkConnectionIndex>
4788                                 <IsConnected>true</IsConnected>
4789                                 <IpAddressAllocationMode>{}</IpAddressAllocationMode>
4790                                 <NetworkAdapterType>{}</NetworkAdapterType>
4791                                 </NetworkConnection>""".format(primary_nic_index, network_name, nicIndex,
4792                                                                                allocation_mode, nic_type)
4793                         # Stub for ip_address feature
4794                         if ip_address:
4795                             ip_tag = '<IpAddress>{}</IpAddress>'.format(ip_address)
4796                             item =  item.replace('</NetworkConnectionIndex>\n','</NetworkConnectionIndex>\n{}\n'.format(ip_tag))
4797
4798                         if mac_address:
4799                             mac_tag = '<MACAddress>{}</MACAddress>'.format(mac_address)
4800                             item = item.replace('</IsConnected>\n','</IsConnected>\n{}\n'.format(mac_tag))
4801
4802                         data = data.replace('</ovf:Info>\n','</ovf:Info>\n{}\n</NetworkConnectionSection>'.format(item))
4803                     else:
4804                         new_item = """<NetworkConnection network="{}">
4805                                     <NetworkConnectionIndex>{}</NetworkConnectionIndex>
4806                                     <IsConnected>true</IsConnected>
4807                                     <IpAddressAllocationMode>{}</IpAddressAllocationMode>
4808                                     <NetworkAdapterType>{}</NetworkAdapterType>
4809                                     </NetworkConnection>""".format(network_name, nicIndex,
4810                                                                 allocation_mode, nic_type)
4811                         # Stub for ip_address feature
4812                         if ip_address:
4813                             ip_tag = '<IpAddress>{}</IpAddress>'.format(ip_address)
4814                             new_item =  new_item.replace('</NetworkConnectionIndex>\n','</NetworkConnectionIndex>\n{}\n'.format(ip_tag))
4815
4816                         if mac_address:
4817                             mac_tag = '<MACAddress>{}</MACAddress>'.format(mac_address)
4818                             new_item =  new_item.replace('</IsConnected>\n','</IsConnected>\n{}\n'.format(mac_tag))
4819
4820                         data = data + new_item + '</NetworkConnectionSection>'
4821
4822                     headers['Content-Type'] = 'application/vnd.vmware.vcloud.networkConnectionSection+xml'
4823
4824                     response = self.perform_request(req_type='PUT',
4825                                                     url=url_rest_call,
4826                                                     headers=headers,
4827                                                     data=data)
4828
4829                     if response.status_code == 403:
4830                         add_headers = {'Content-Type': headers['Content-Type']}
4831                         response = self.retry_rest('PUT', url_rest_call, add_headers, data)
4832
4833                     if response.status_code != 202:
4834                         self.logger.error("REST call {} failed reason : {}"\
4835                                             "status code : {}".format(url_rest_call,
4836                                                                    response.content,
4837                                                               response.status_code))
4838                         raise vimconn.vimconnException("add_network_adapter_to_vms : Failed to update "\
4839                                                                            "network connection section")
4840                     else:
4841                         nic_task = self.get_task_from_response(response.content)
4842                         result = self.client.get_task_monitor().wait_for_success(task=nic_task)
4843                         if result.get('status') == 'success':
4844                             self.logger.info("add_network_adapter_to_vms(): VM {} "\
4845                                                "conneced to NIC type {}".format(vm_id, nic_type))
4846                         else:
4847                             self.logger.error("add_network_adapter_to_vms(): VM {} "\
4848                                                "failed to connect NIC type {}".format(vm_id, nic_type))
4849         except Exception as exp:
4850             self.logger.error("add_network_adapter_to_vms() : exception occurred "\
4851                                                "while adding Network adapter")
4852             raise vimconn.vimconnException(message=exp)
4853
4854
4855     def set_numa_affinity(self, vmuuid, paired_threads_id):
4856         """
4857             Method to assign numa affinity in vm configuration parammeters
4858             Args :
4859                 vmuuid - vm uuid
4860                 paired_threads_id - one or more virtual processor
4861                                     numbers
4862             Returns:
4863                 return if True
4864         """
4865         try:
4866             vcenter_conect, content = self.get_vcenter_content()
4867             vm_moref_id = self.get_vm_moref_id(vmuuid)
4868
4869             host_obj, vm_obj = self.get_vm_obj(content ,vm_moref_id)
4870             if vm_obj:
4871                 config_spec = vim.vm.ConfigSpec()
4872                 config_spec.extraConfig = []
4873                 opt = vim.option.OptionValue()
4874                 opt.key = 'numa.nodeAffinity'
4875                 opt.value = str(paired_threads_id)
4876                 config_spec.extraConfig.append(opt)
4877                 task = vm_obj.ReconfigVM_Task(config_spec)
4878                 if task:
4879                     result = self.wait_for_vcenter_task(task, vcenter_conect)
4880                     extra_config = vm_obj.config.extraConfig
4881                     flag = False
4882                     for opts in extra_config:
4883                         if 'numa.nodeAffinity' in opts.key:
4884                             flag = True
4885                             self.logger.info("set_numa_affinity: Sucessfully assign numa affinity "\
4886                                                      "value {} for vm {}".format(opt.value, vm_obj))
4887                         if flag:
4888                             return
4889             else:
4890                 self.logger.error("set_numa_affinity: Failed to assign numa affinity")
4891         except Exception as exp:
4892             self.logger.error("set_numa_affinity : exception occurred while setting numa affinity "\
4893                                                        "for VM {} : {}".format(vm_obj, vm_moref_id))
4894             raise vimconn.vimconnException("set_numa_affinity : Error {} failed to assign numa "\
4895                                                                            "affinity".format(exp))
4896
4897
4898     def cloud_init(self, vapp, cloud_config):
4899         """
4900         Method to inject ssh-key
4901         vapp - vapp object
4902         cloud_config a dictionary with:
4903                 'key-pairs': (optional) list of strings with the public key to be inserted to the default user
4904                 'users': (optional) list of users to be inserted, each item is a dict with:
4905                     'name': (mandatory) user name,
4906                     'key-pairs': (optional) list of strings with the public key to be inserted to the user
4907                 'user-data': (optional) can be a string with the text script to be passed directly to cloud-init,
4908                     or a list of strings, each one contains a script to be passed, usually with a MIMEmultipart file
4909                 'config-files': (optional). List of files to be transferred. Each item is a dict with:
4910                     'dest': (mandatory) string with the destination absolute path
4911                     'encoding': (optional, by default text). Can be one of:
4912                         'b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64'
4913                     'content' (mandatory): string with the content of the file
4914                     'permissions': (optional) string with file permissions, typically octal notation '0644'
4915                     'owner': (optional) file owner, string with the format 'owner:group'
4916                 'boot-data-drive': boolean to indicate if user-data must be passed using a boot drive (hard disk
4917         """
4918         try:
4919             if not isinstance(cloud_config, dict):
4920                 raise Exception("cloud_init : parameter cloud_config is not a dictionary")
4921             else:
4922                 key_pairs = []
4923                 userdata = []
4924                 if "key-pairs" in cloud_config:
4925                     key_pairs = cloud_config["key-pairs"]
4926
4927                 if "users" in cloud_config:
4928                     userdata = cloud_config["users"]
4929
4930                 self.logger.debug("cloud_init : Guest os customization started..")
4931                 customize_script = self.format_script(key_pairs=key_pairs, users_list=userdata)
4932                 customize_script = customize_script.replace("&","&amp;")
4933                 self.guest_customization(vapp, customize_script)
4934
4935         except Exception as exp:
4936             self.logger.error("cloud_init : exception occurred while injecting "\
4937                                                                        "ssh-key")
4938             raise vimconn.vimconnException("cloud_init : Error {} failed to inject "\
4939                                                                "ssh-key".format(exp))
4940
4941     def format_script(self, key_pairs=[], users_list=[]):
4942         bash_script = """#!/bin/sh
4943         echo performing customization tasks with param $1 at `date "+DATE: %Y-%m-%d - TIME: %H:%M:%S"` >> /root/customization.log
4944         if [ "$1" = "precustomization" ];then
4945             echo performing precustomization tasks   on `date "+DATE: %Y-%m-%d - TIME: %H:%M:%S"` >> /root/customization.log
4946         """
4947
4948         keys = "\n".join(key_pairs)
4949         if keys:
4950             keys_data = """
4951             if [ ! -d /root/.ssh ];then
4952                 mkdir /root/.ssh
4953                 chown root:root /root/.ssh
4954                 chmod 700 /root/.ssh
4955                 touch /root/.ssh/authorized_keys
4956                 chown root:root /root/.ssh/authorized_keys
4957                 chmod 600 /root/.ssh/authorized_keys
4958                 # make centos with selinux happy
4959                 which restorecon && restorecon -Rv /root/.ssh
4960             else
4961                 touch /root/.ssh/authorized_keys
4962                 chown root:root /root/.ssh/authorized_keys
4963                 chmod 600 /root/.ssh/authorized_keys
4964             fi
4965             echo '{key}' >> /root/.ssh/authorized_keys
4966             """.format(key=keys)
4967
4968             bash_script+= keys_data
4969
4970         for user in users_list:
4971             if 'name' in user: user_name = user['name']
4972             if 'key-pairs' in user:
4973                 user_keys = "\n".join(user['key-pairs'])
4974             else:
4975                 user_keys = None
4976
4977             add_user_name = """
4978                 useradd -d /home/{user_name} -m -g users -s /bin/bash {user_name}
4979                 """.format(user_name=user_name)
4980
4981             bash_script+= add_user_name
4982
4983             if user_keys:
4984                 user_keys_data = """
4985                 mkdir /home/{user_name}/.ssh
4986                 chown {user_name}:{user_name} /home/{user_name}/.ssh
4987                 chmod 700 /home/{user_name}/.ssh
4988                 touch /home/{user_name}/.ssh/authorized_keys
4989                 chown {user_name}:{user_name} /home/{user_name}/.ssh/authorized_keys
4990                 chmod 600 /home/{user_name}/.ssh/authorized_keys
4991                 # make centos with selinux happy
4992                 which restorecon && restorecon -Rv /home/{user_name}/.ssh
4993                 echo '{user_key}' >> /home/{user_name}/.ssh/authorized_keys
4994                 """.format(user_name=user_name,user_key=user_keys)
4995
4996                 bash_script+= user_keys_data
4997
4998         return bash_script+"\n\tfi"
4999
5000     def guest_customization(self, vapp, customize_script):
5001         """
5002         Method to customize guest os
5003         vapp - Vapp object
5004         customize_script - Customize script to be run at first boot of VM.
5005         """
5006         for vm in vapp.get_all_vms():
5007             vm_id = vm.get('id').split(':')[-1]
5008             vm_name = vm.get('name')
5009             vm_name = vm_name.replace('_','-')
5010
5011             vm_customization_url = "{}/api/vApp/vm-{}/guestCustomizationSection/".format(self.url, vm_id)
5012             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
5013                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
5014
5015             headers['Content-Type'] = "application/vnd.vmware.vcloud.guestCustomizationSection+xml"
5016
5017             data = """<GuestCustomizationSection
5018                            xmlns="http://www.vmware.com/vcloud/v1.5"
5019                            xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/1"
5020                            ovf:required="false" href="{}" type="application/vnd.vmware.vcloud.guestCustomizationSection+xml">
5021                            <ovf:Info>Specifies Guest OS Customization Settings</ovf:Info>
5022                            <Enabled>true</Enabled>
5023                            <ChangeSid>false</ChangeSid>
5024                            <VirtualMachineId>{}</VirtualMachineId>
5025                            <JoinDomainEnabled>false</JoinDomainEnabled>
5026                            <UseOrgSettings>false</UseOrgSettings>
5027                            <AdminPasswordEnabled>false</AdminPasswordEnabled>
5028                            <AdminPasswordAuto>true</AdminPasswordAuto>
5029                            <AdminAutoLogonEnabled>false</AdminAutoLogonEnabled>
5030                            <AdminAutoLogonCount>0</AdminAutoLogonCount>
5031                            <ResetPasswordRequired>false</ResetPasswordRequired>
5032                            <CustomizationScript>{}</CustomizationScript>
5033                            <ComputerName>{}</ComputerName>
5034                            <Link href="{}" type="application/vnd.vmware.vcloud.guestCustomizationSection+xml" rel="edit"/>
5035                        </GuestCustomizationSection>
5036                    """.format(vm_customization_url,
5037                                              vm_id,
5038                                   customize_script,
5039                                            vm_name,
5040                               vm_customization_url)
5041
5042             response = self.perform_request(req_type='PUT',
5043                                              url=vm_customization_url,
5044                                              headers=headers,
5045                                              data=data)
5046             if response.status_code == 202:
5047                 guest_task = self.get_task_from_response(response.content)
5048                 self.client.get_task_monitor().wait_for_success(task=guest_task)
5049                 self.logger.info("guest_customization : customized guest os task "\
5050                                              "completed for VM {}".format(vm_name))
5051             else:
5052                 self.logger.error("guest_customization : task for customized guest os"\
5053                                                     "failed for VM {}".format(vm_name))
5054                 raise vimconn.vimconnException("guest_customization : failed to perform"\
5055                                        "guest os customization on VM {}".format(vm_name))
5056
5057     def add_new_disk(self, vapp_uuid, disk_size):
5058         """
5059             Method to create an empty vm disk
5060
5061             Args:
5062                 vapp_uuid - is vapp identifier.
5063                 disk_size - size of disk to be created in GB
5064
5065             Returns:
5066                 None
5067         """
5068         status = False
5069         vm_details = None
5070         try:
5071             #Disk size in GB, convert it into MB
5072             if disk_size is not None:
5073                 disk_size_mb = int(disk_size) * 1024
5074                 vm_details = self.get_vapp_details_rest(vapp_uuid)
5075
5076             if vm_details and "vm_virtual_hardware" in vm_details:
5077                 self.logger.info("Adding disk to VM: {} disk size:{}GB".format(vm_details["name"], disk_size))
5078                 disk_href = vm_details["vm_virtual_hardware"]["disk_edit_href"]
5079                 status = self.add_new_disk_rest(disk_href, disk_size_mb)
5080
5081         except Exception as exp:
5082             msg = "Error occurred while creating new disk {}.".format(exp)
5083             self.rollback_newvm(vapp_uuid, msg)
5084
5085         if status:
5086             self.logger.info("Added new disk to VM: {} disk size:{}GB".format(vm_details["name"], disk_size))
5087         else:
5088             #If failed to add disk, delete VM
5089             msg = "add_new_disk: Failed to add new disk to {}".format(vm_details["name"])
5090             self.rollback_newvm(vapp_uuid, msg)
5091
5092
5093     def add_new_disk_rest(self, disk_href, disk_size_mb):
5094         """
5095         Retrives vApp Disks section & add new empty disk
5096
5097         Args:
5098             disk_href: Disk section href to addd disk
5099             disk_size_mb: Disk size in MB
5100
5101             Returns: Status of add new disk task
5102         """
5103         status = False
5104         if self.client._session:
5105             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
5106                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
5107             response = self.perform_request(req_type='GET',
5108                                             url=disk_href,
5109                                             headers=headers)
5110
5111         if response.status_code == 403:
5112             response = self.retry_rest('GET', disk_href)
5113
5114         if response.status_code != requests.codes.ok:
5115             self.logger.error("add_new_disk_rest: GET REST API call {} failed. Return status code {}"
5116                               .format(disk_href, response.status_code))
5117             return status
5118         try:
5119             #Find but type & max of instance IDs assigned to disks
5120             lxmlroot_respond = lxmlElementTree.fromstring(response.content)
5121             namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.iteritems() if prefix}
5122             #For python3
5123             #namespaces = {prefix:uri for prefix,uri in lxmlroot_respond.nsmap.items() if prefix}
5124             namespaces["xmlns"]= "http://www.vmware.com/vcloud/v1.5"
5125             instance_id = 0
5126             for item in lxmlroot_respond.iterfind('xmlns:Item',namespaces):
5127                 if item.find("rasd:Description",namespaces).text == "Hard disk":
5128                     inst_id = int(item.find("rasd:InstanceID" ,namespaces).text)
5129                     if inst_id > instance_id:
5130                         instance_id = inst_id
5131                         disk_item = item.find("rasd:HostResource" ,namespaces)
5132                         bus_subtype = disk_item.attrib["{"+namespaces['xmlns']+"}busSubType"]
5133                         bus_type = disk_item.attrib["{"+namespaces['xmlns']+"}busType"]
5134
5135             instance_id = instance_id + 1
5136             new_item =   """<Item>
5137                                 <rasd:Description>Hard disk</rasd:Description>
5138                                 <rasd:ElementName>New disk</rasd:ElementName>
5139                                 <rasd:HostResource
5140                                     xmlns:vcloud="http://www.vmware.com/vcloud/v1.5"
5141                                     vcloud:capacity="{}"
5142                                     vcloud:busSubType="{}"
5143                                     vcloud:busType="{}"></rasd:HostResource>
5144                                 <rasd:InstanceID>{}</rasd:InstanceID>
5145                                 <rasd:ResourceType>17</rasd:ResourceType>
5146                             </Item>""".format(disk_size_mb, bus_subtype, bus_type, instance_id)
5147
5148             new_data = response.content
5149             #Add new item at the bottom
5150             new_data = new_data.replace('</Item>\n</RasdItemsList>', '</Item>\n{}\n</RasdItemsList>'.format(new_item))
5151
5152             # Send PUT request to modify virtual hardware section with new disk
5153             headers['Content-Type'] = 'application/vnd.vmware.vcloud.rasdItemsList+xml; charset=ISO-8859-1'
5154
5155             response = self.perform_request(req_type='PUT',
5156                                             url=disk_href,
5157                                             data=new_data,
5158                                             headers=headers)
5159
5160             if response.status_code == 403:
5161                 add_headers = {'Content-Type': headers['Content-Type']}
5162                 response = self.retry_rest('PUT', disk_href, add_headers, new_data)
5163
5164             if response.status_code != 202:
5165                 self.logger.error("PUT REST API call {} failed. Return status code {}. Response Content:{}"
5166                                   .format(disk_href, response.status_code, response.content))
5167             else:
5168                 add_disk_task = self.get_task_from_response(response.content)
5169                 result = self.client.get_task_monitor().wait_for_success(task=add_disk_task)
5170                 if result.get('status') == 'success':
5171                     status = True
5172                 else:
5173                     self.logger.error("Add new disk REST task failed to add {} MB disk".format(disk_size_mb))
5174
5175         except Exception as exp:
5176             self.logger.error("Error occurred calling rest api for creating new disk {}".format(exp))
5177
5178         return status
5179
5180
5181     def add_existing_disk(self, catalogs=None, image_id=None, size=None, template_name=None, vapp_uuid=None):
5182         """
5183             Method to add existing disk to vm
5184             Args :
5185                 catalogs - List of VDC catalogs
5186                 image_id - Catalog ID
5187                 template_name - Name of template in catalog
5188                 vapp_uuid - UUID of vApp
5189             Returns:
5190                 None
5191         """
5192         disk_info = None
5193         vcenter_conect, content = self.get_vcenter_content()
5194         #find moref-id of vm in image
5195         catalog_vm_info = self.get_vapp_template_details(catalogs=catalogs,
5196                                                          image_id=image_id,
5197                                                         )
5198
5199         if catalog_vm_info and "vm_vcenter_info" in catalog_vm_info:
5200             if "vm_moref_id" in catalog_vm_info["vm_vcenter_info"]:
5201                 catalog_vm_moref_id = catalog_vm_info["vm_vcenter_info"].get("vm_moref_id", None)
5202                 if catalog_vm_moref_id:
5203                     self.logger.info("Moref_id of VM in catalog : {}" .format(catalog_vm_moref_id))
5204                     host, catalog_vm_obj = self.get_vm_obj(content, catalog_vm_moref_id)
5205                     if catalog_vm_obj:
5206                         #find existing disk
5207                         disk_info = self.find_disk(catalog_vm_obj)
5208                     else:
5209                         exp_msg = "No VM with image id {} found".format(image_id)
5210                         self.rollback_newvm(vapp_uuid, exp_msg, exp_type="NotFound")
5211         else:
5212             exp_msg = "No Image found with image ID {} ".format(image_id)
5213             self.rollback_newvm(vapp_uuid, exp_msg, exp_type="NotFound")
5214
5215         if disk_info:
5216             self.logger.info("Existing disk_info : {}".format(disk_info))
5217             #get VM
5218             vm_moref_id = self.get_vm_moref_id(vapp_uuid)
5219             host, vm_obj = self.get_vm_obj(content, vm_moref_id)
5220             if vm_obj:
5221                 status = self.add_disk(vcenter_conect=vcenter_conect,
5222                                        vm=vm_obj,
5223                                        disk_info=disk_info,
5224                                        size=size,
5225                                        vapp_uuid=vapp_uuid
5226                                        )
5227             if status:
5228                 self.logger.info("Disk from image id {} added to {}".format(image_id,
5229                                                                             vm_obj.config.name)
5230                                  )
5231         else:
5232             msg = "No disk found with image id {} to add in VM {}".format(
5233                                                             image_id,
5234                                                             vm_obj.config.name)
5235             self.rollback_newvm(vapp_uuid, msg, exp_type="NotFound")
5236
5237
5238     def find_disk(self, vm_obj):
5239         """
5240          Method to find details of existing disk in VM
5241             Args :
5242                 vm_obj - vCenter object of VM
5243                 image_id - Catalog ID
5244             Returns:
5245                 disk_info : dict of disk details
5246         """
5247         disk_info = {}
5248         if vm_obj:
5249             try:
5250                 devices = vm_obj.config.hardware.device
5251                 for device in devices:
5252                     if type(device) is vim.vm.device.VirtualDisk:
5253                         if isinstance(device.backing,vim.vm.device.VirtualDisk.FlatVer2BackingInfo) and hasattr(device.backing, 'fileName'):
5254                             disk_info["full_path"] = device.backing.fileName
5255                             disk_info["datastore"] = device.backing.datastore
5256                             disk_info["capacityKB"] = device.capacityInKB
5257                             break
5258             except Exception as exp:
5259                 self.logger.error("find_disk() : exception occurred while "\
5260                                   "getting existing disk details :{}".format(exp))
5261         return disk_info
5262
5263
5264     def add_disk(self, vcenter_conect=None, vm=None, size=None, vapp_uuid=None, disk_info={}):
5265         """
5266          Method to add existing disk in VM
5267             Args :
5268                 vcenter_conect - vCenter content object
5269                 vm - vCenter vm object
5270                 disk_info : dict of disk details
5271             Returns:
5272                 status : status of add disk task
5273         """
5274         datastore = disk_info["datastore"] if "datastore" in disk_info else None
5275         fullpath = disk_info["full_path"] if "full_path" in disk_info else None
5276         capacityKB = disk_info["capacityKB"] if "capacityKB" in disk_info else None
5277         if size is not None:
5278             #Convert size from GB to KB
5279             sizeKB = int(size) * 1024 * 1024
5280             #compare size of existing disk and user given size.Assign whicherver is greater
5281             self.logger.info("Add Existing disk : sizeKB {} , capacityKB {}".format(
5282                                                                     sizeKB, capacityKB))
5283             if sizeKB > capacityKB:
5284                 capacityKB = sizeKB
5285
5286         if datastore and fullpath and capacityKB:
5287             try:
5288                 spec = vim.vm.ConfigSpec()
5289                 # get all disks on a VM, set unit_number to the next available
5290                 unit_number = 0
5291                 for dev in vm.config.hardware.device:
5292                     if hasattr(dev.backing, 'fileName'):
5293                         unit_number = int(dev.unitNumber) + 1
5294                         # unit_number 7 reserved for scsi controller
5295                         if unit_number == 7:
5296                             unit_number += 1
5297                     if isinstance(dev, vim.vm.device.VirtualDisk):
5298                         #vim.vm.device.VirtualSCSIController
5299                         controller_key = dev.controllerKey
5300
5301                 self.logger.info("Add Existing disk : unit number {} , controller key {}".format(
5302                                                                     unit_number, controller_key))
5303                 # add disk here
5304                 dev_changes = []
5305                 disk_spec = vim.vm.device.VirtualDeviceSpec()
5306                 disk_spec.operation = vim.vm.device.VirtualDeviceSpec.Operation.add
5307                 disk_spec.device = vim.vm.device.VirtualDisk()
5308                 disk_spec.device.backing = \
5309                     vim.vm.device.VirtualDisk.FlatVer2BackingInfo()
5310                 disk_spec.device.backing.thinProvisioned = True
5311                 disk_spec.device.backing.diskMode = 'persistent'
5312                 disk_spec.device.backing.datastore  = datastore
5313                 disk_spec.device.backing.fileName  = fullpath
5314
5315                 disk_spec.device.unitNumber = unit_number
5316                 disk_spec.device.capacityInKB = capacityKB
5317                 disk_spec.device.controllerKey = controller_key
5318                 dev_changes.append(disk_spec)
5319                 spec.deviceChange = dev_changes
5320                 task = vm.ReconfigVM_Task(spec=spec)
5321                 status = self.wait_for_vcenter_task(task, vcenter_conect)
5322                 return status
5323             except Exception as exp:
5324                 exp_msg = "add_disk() : exception {} occurred while adding disk "\
5325                           "{} to vm {}".format(exp,
5326                                                fullpath,
5327                                                vm.config.name)
5328                 self.rollback_newvm(vapp_uuid, exp_msg)
5329         else:
5330             msg = "add_disk() : Can not add disk to VM with disk info {} ".format(disk_info)
5331             self.rollback_newvm(vapp_uuid, msg)
5332
5333
5334     def get_vcenter_content(self):
5335         """
5336          Get the vsphere content object
5337         """
5338         try:
5339             vm_vcenter_info = self.get_vm_vcenter_info()
5340         except Exception as exp:
5341             self.logger.error("Error occurred while getting vCenter infromationn"\
5342                              " for VM : {}".format(exp))
5343             raise vimconn.vimconnException(message=exp)
5344
5345         context = None
5346         if hasattr(ssl, '_create_unverified_context'):
5347             context = ssl._create_unverified_context()
5348
5349         vcenter_conect = SmartConnect(
5350                     host=vm_vcenter_info["vm_vcenter_ip"],
5351                     user=vm_vcenter_info["vm_vcenter_user"],
5352                     pwd=vm_vcenter_info["vm_vcenter_password"],
5353                     port=int(vm_vcenter_info["vm_vcenter_port"]),
5354                     sslContext=context
5355                 )
5356         atexit.register(Disconnect, vcenter_conect)
5357         content = vcenter_conect.RetrieveContent()
5358         return vcenter_conect, content
5359
5360
5361     def get_vm_moref_id(self, vapp_uuid):
5362         """
5363         Get the moref_id of given VM
5364         """
5365         try:
5366             if vapp_uuid:
5367                 vm_details = self.get_vapp_details_rest(vapp_uuid, need_admin_access=True)
5368                 if vm_details and "vm_vcenter_info" in vm_details:
5369                     vm_moref_id = vm_details["vm_vcenter_info"].get("vm_moref_id", None)
5370             return vm_moref_id
5371
5372         except Exception as exp:
5373             self.logger.error("Error occurred while getting VM moref ID "\
5374                              " for VM : {}".format(exp))
5375             return None
5376
5377
5378     def get_vapp_template_details(self, catalogs=None, image_id=None , template_name=None):
5379         """
5380             Method to get vApp template details
5381                 Args :
5382                     catalogs - list of VDC catalogs
5383                     image_id - Catalog ID to find
5384                     template_name : template name in catalog
5385                 Returns:
5386                     parsed_respond : dict of vApp tempalte details
5387         """
5388         parsed_response = {}
5389
5390         vca = self.connect_as_admin()
5391         if not vca:
5392             raise vimconn.vimconnConnectionException("Failed to connect vCD")
5393
5394         try:
5395             org, vdc = self.get_vdc_details()
5396             catalog = self.get_catalog_obj(image_id, catalogs)
5397             if catalog:
5398                 items = org.get_catalog_item(catalog.get('name'), catalog.get('name'))
5399                 catalog_items = [items.attrib]
5400
5401                 if len(catalog_items) == 1:
5402                     headers = {'Accept':'application/*+xml;version=' + API_VERSION,
5403                            'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
5404
5405                     response = self.perform_request(req_type='GET',
5406                                                     url=catalog_items[0].get('href'),
5407                                                     headers=headers)
5408                     catalogItem = XmlElementTree.fromstring(response.content)
5409                     entity = [child for child in catalogItem if child.get("type") == "application/vnd.vmware.vcloud.vAppTemplate+xml"][0]
5410                     vapp_tempalte_href = entity.get("href")
5411                     #get vapp details and parse moref id
5412
5413                     namespaces = {"vssd":"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData" ,
5414                                   'ovf': 'http://schemas.dmtf.org/ovf/envelope/1',
5415                                   'vmw': 'http://www.vmware.com/schema/ovf',
5416                                   'vm': 'http://www.vmware.com/vcloud/v1.5',
5417                                   'rasd':"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData",
5418                                   'vmext':"http://www.vmware.com/vcloud/extension/v1.5",
5419                                   'xmlns':"http://www.vmware.com/vcloud/v1.5"
5420                                 }
5421
5422                     if vca._session:
5423                         response = self.perform_request(req_type='GET',
5424                                                     url=vapp_tempalte_href,
5425                                                     headers=headers)
5426
5427                         if response.status_code != requests.codes.ok:
5428                             self.logger.debug("REST API call {} failed. Return status code {}".format(
5429                                                 vapp_tempalte_href, response.status_code))
5430
5431                         else:
5432                             xmlroot_respond = XmlElementTree.fromstring(response.content)
5433                             children_section = xmlroot_respond.find('vm:Children/', namespaces)
5434                             if children_section is not None:
5435                                 vCloud_extension_section = children_section.find('xmlns:VCloudExtension', namespaces)
5436                             if vCloud_extension_section is not None:
5437                                 vm_vcenter_info = {}
5438                                 vim_info = vCloud_extension_section.find('vmext:VmVimInfo', namespaces)
5439                                 vmext = vim_info.find('vmext:VmVimObjectRef', namespaces)
5440                                 if vmext is not None:
5441                                     vm_vcenter_info["vm_moref_id"] = vmext.find('vmext:MoRef', namespaces).text
5442                                 parsed_response["vm_vcenter_info"]= vm_vcenter_info
5443
5444         except Exception as exp :
5445             self.logger.info("Error occurred calling rest api for getting vApp details {}".format(exp))
5446
5447         return parsed_response
5448
5449
5450     def rollback_newvm(self, vapp_uuid, msg , exp_type="Genric"):
5451         """
5452             Method to delete vApp
5453                 Args :
5454                     vapp_uuid - vApp UUID
5455                     msg - Error message to be logged
5456                     exp_type : Exception type
5457                 Returns:
5458                     None
5459         """
5460         if vapp_uuid:
5461             status = self.delete_vminstance(vapp_uuid)
5462         else:
5463             msg = "No vApp ID"
5464         self.logger.error(msg)
5465         if exp_type == "Genric":
5466             raise vimconn.vimconnException(msg)
5467         elif exp_type == "NotFound":
5468             raise vimconn.vimconnNotFoundException(message=msg)
5469
5470     def add_sriov(self, vapp_uuid, sriov_nets, vmname_andid):
5471         """
5472             Method to attach SRIOV adapters to VM
5473
5474              Args:
5475                 vapp_uuid - uuid of vApp/VM
5476                 sriov_nets - SRIOV devices infromation as specified in VNFD (flavor)
5477                 vmname_andid - vmname
5478
5479             Returns:
5480                 The status of add SRIOV adapter task , vm object and
5481                 vcenter_conect object
5482         """
5483         vm_obj = None
5484         vcenter_conect, content = self.get_vcenter_content()
5485         vm_moref_id = self.get_vm_moref_id(vapp_uuid)
5486
5487         if vm_moref_id:
5488             try:
5489                 no_of_sriov_devices = len(sriov_nets)
5490                 if no_of_sriov_devices > 0:
5491                     #Get VM and its host
5492                     host_obj, vm_obj = self.get_vm_obj(content, vm_moref_id)
5493                     self.logger.info("VM {} is currently on host {}".format(vm_obj, host_obj))
5494                     if host_obj and vm_obj:
5495                         #get SRIOV devies from host on which vapp is currently installed
5496                         avilable_sriov_devices = self.get_sriov_devices(host_obj,
5497                                                                 no_of_sriov_devices,
5498                                                                 )
5499
5500                         if len(avilable_sriov_devices) == 0:
5501                             #find other hosts with active pci devices
5502                             new_host_obj , avilable_sriov_devices = self.get_host_and_sriov_devices(
5503                                                                 content,
5504                                                                 no_of_sriov_devices,
5505                                                                 )
5506
5507                             if new_host_obj is not None and len(avilable_sriov_devices)> 0:
5508                                 #Migrate vm to the host where SRIOV devices are available
5509                                 self.logger.info("Relocate VM {} on new host {}".format(vm_obj,
5510                                                                                     new_host_obj))
5511                                 task = self.relocate_vm(new_host_obj, vm_obj)
5512                                 if task is not None:
5513                                     result = self.wait_for_vcenter_task(task, vcenter_conect)
5514                                     self.logger.info("Migrate VM status: {}".format(result))
5515                                     host_obj = new_host_obj
5516                                 else:
5517                                     self.logger.info("Fail to migrate VM : {}".format(result))
5518                                     raise vimconn.vimconnNotFoundException(
5519                                     "Fail to migrate VM : {} to host {}".format(
5520                                                     vmname_andid,
5521                                                     new_host_obj)
5522                                         )
5523
5524                         if host_obj is not None and avilable_sriov_devices is not None and len(avilable_sriov_devices)> 0:
5525                             #Add SRIOV devices one by one
5526                             for sriov_net in sriov_nets:
5527                                 network_name = sriov_net.get('net_id')
5528                                 dvs_portgr_name = self.create_dvPort_group(network_name)
5529                                 if sriov_net.get('type') == "VF" or sriov_net.get('type') == "SR-IOV":
5530                                     #add vlan ID ,Modify portgroup for vlan ID
5531                                     self.configure_vlanID(content, vcenter_conect, network_name)
5532
5533                                 task = self.add_sriov_to_vm(content,
5534                                                             vm_obj,
5535                                                             host_obj,
5536                                                             network_name,
5537                                                             avilable_sriov_devices[0]
5538                                                             )
5539                                 if task:
5540                                     status= self.wait_for_vcenter_task(task, vcenter_conect)
5541                                     if status:
5542                                         self.logger.info("Added SRIOV {} to VM {}".format(
5543                                                                         no_of_sriov_devices,
5544                                                                         str(vm_obj)))
5545                                 else:
5546                                     self.logger.error("Fail to add SRIOV {} to VM {}".format(
5547                                                                         no_of_sriov_devices,
5548                                                                         str(vm_obj)))
5549                                     raise vimconn.vimconnUnexpectedResponse(
5550                                     "Fail to add SRIOV adapter in VM ".format(str(vm_obj))
5551                                         )
5552                             return True, vm_obj, vcenter_conect
5553                         else:
5554                             self.logger.error("Currently there is no host with"\
5555                                               " {} number of avaialble SRIOV "\
5556                                               "VFs required for VM {}".format(
5557                                                                 no_of_sriov_devices,
5558                                                                 vmname_andid)
5559                                               )
5560                             raise vimconn.vimconnNotFoundException(
5561                                     "Currently there is no host with {} "\
5562                                     "number of avaialble SRIOV devices required for VM {}".format(
5563                                                                             no_of_sriov_devices,
5564                                                                             vmname_andid))
5565                 else:
5566                     self.logger.debug("No infromation about SRIOV devices {} ",sriov_nets)
5567
5568             except vmodl.MethodFault as error:
5569                 self.logger.error("Error occurred while adding SRIOV {} ",error)
5570         return None, vm_obj, vcenter_conect
5571
5572
5573     def get_sriov_devices(self,host, no_of_vfs):
5574         """
5575             Method to get the details of SRIOV devices on given host
5576              Args:
5577                 host - vSphere host object
5578                 no_of_vfs - number of VFs needed on host
5579
5580              Returns:
5581                 array of SRIOV devices
5582         """
5583         sriovInfo=[]
5584         if host:
5585             for device in host.config.pciPassthruInfo:
5586                 if isinstance(device,vim.host.SriovInfo) and device.sriovActive:
5587                     if device.numVirtualFunction >= no_of_vfs:
5588                         sriovInfo.append(device)
5589                         break
5590         return sriovInfo
5591
5592
5593     def get_host_and_sriov_devices(self, content, no_of_vfs):
5594         """
5595          Method to get the details of SRIOV devices infromation on all hosts
5596
5597             Args:
5598                 content - vSphere host object
5599                 no_of_vfs - number of pci VFs needed on host
5600
5601             Returns:
5602                  array of SRIOV devices and host object
5603         """
5604         host_obj = None
5605         sriov_device_objs = None
5606         try:
5607             if content:
5608                 container = content.viewManager.CreateContainerView(content.rootFolder,
5609                                                             [vim.HostSystem], True)
5610                 for host in container.view:
5611                     devices = self.get_sriov_devices(host, no_of_vfs)
5612                     if devices:
5613                         host_obj = host
5614                         sriov_device_objs = devices
5615                         break
5616         except Exception as exp:
5617             self.logger.error("Error {} occurred while finding SRIOV devices on host: {}".format(exp, host_obj))
5618
5619         return host_obj,sriov_device_objs
5620
5621
5622     def add_sriov_to_vm(self,content, vm_obj, host_obj, network_name, sriov_device):
5623         """
5624          Method to add SRIOV adapter to vm
5625
5626             Args:
5627                 host_obj - vSphere host object
5628                 vm_obj - vSphere vm object
5629                 content - vCenter content object
5630                 network_name - name of distributed virtaul portgroup
5631                 sriov_device - SRIOV device info
5632
5633             Returns:
5634                  task object
5635         """
5636         devices = []
5637         vnic_label = "sriov nic"
5638         try:
5639             dvs_portgr = self.get_dvport_group(network_name)
5640             network_name = dvs_portgr.name
5641             nic = vim.vm.device.VirtualDeviceSpec()
5642             # VM device
5643             nic.operation = vim.vm.device.VirtualDeviceSpec.Operation.add
5644             nic.device = vim.vm.device.VirtualSriovEthernetCard()
5645             nic.device.addressType = 'assigned'
5646             #nic.device.key = 13016
5647             nic.device.deviceInfo = vim.Description()
5648             nic.device.deviceInfo.label = vnic_label
5649             nic.device.deviceInfo.summary = network_name
5650             nic.device.backing = vim.vm.device.VirtualEthernetCard.NetworkBackingInfo()
5651
5652             nic.device.backing.network = self.get_obj(content, [vim.Network], network_name)
5653             nic.device.backing.deviceName = network_name
5654             nic.device.backing.useAutoDetect = False
5655             nic.device.connectable = vim.vm.device.VirtualDevice.ConnectInfo()
5656             nic.device.connectable.startConnected = True
5657             nic.device.connectable.allowGuestControl = True
5658
5659             nic.device.sriovBacking = vim.vm.device.VirtualSriovEthernetCard.SriovBackingInfo()
5660             nic.device.sriovBacking.physicalFunctionBacking = vim.vm.device.VirtualPCIPassthrough.DeviceBackingInfo()
5661             nic.device.sriovBacking.physicalFunctionBacking.id = sriov_device.id
5662
5663             devices.append(nic)
5664             vmconf = vim.vm.ConfigSpec(deviceChange=devices)
5665             task = vm_obj.ReconfigVM_Task(vmconf)
5666             return task
5667         except Exception as exp:
5668             self.logger.error("Error {} occurred while adding SRIOV adapter in VM: {}".format(exp, vm_obj))
5669             return None
5670
5671
5672     def create_dvPort_group(self, network_name):
5673         """
5674          Method to create disributed virtual portgroup
5675
5676             Args:
5677                 network_name - name of network/portgroup
5678
5679             Returns:
5680                 portgroup key
5681         """
5682         try:
5683             new_network_name = [network_name, '-', str(uuid.uuid4())]
5684             network_name=''.join(new_network_name)
5685             vcenter_conect, content = self.get_vcenter_content()
5686
5687             dv_switch = self.get_obj(content, [vim.DistributedVirtualSwitch], self.dvs_name)
5688             if dv_switch:
5689                 dv_pg_spec = vim.dvs.DistributedVirtualPortgroup.ConfigSpec()
5690                 dv_pg_spec.name = network_name
5691
5692                 dv_pg_spec.type = vim.dvs.DistributedVirtualPortgroup.PortgroupType.earlyBinding
5693                 dv_pg_spec.defaultPortConfig = vim.dvs.VmwareDistributedVirtualSwitch.VmwarePortConfigPolicy()
5694                 dv_pg_spec.defaultPortConfig.securityPolicy = vim.dvs.VmwareDistributedVirtualSwitch.SecurityPolicy()
5695                 dv_pg_spec.defaultPortConfig.securityPolicy.allowPromiscuous = vim.BoolPolicy(value=False)
5696                 dv_pg_spec.defaultPortConfig.securityPolicy.forgedTransmits = vim.BoolPolicy(value=False)
5697                 dv_pg_spec.defaultPortConfig.securityPolicy.macChanges = vim.BoolPolicy(value=False)
5698
5699                 task = dv_switch.AddDVPortgroup_Task([dv_pg_spec])
5700                 self.wait_for_vcenter_task(task, vcenter_conect)
5701
5702                 dvPort_group = self.get_obj(content, [vim.dvs.DistributedVirtualPortgroup], network_name)
5703                 if dvPort_group:
5704                     self.logger.info("Created disributed virtaul port group: {}".format(dvPort_group))
5705                     return dvPort_group.key
5706             else:
5707                 self.logger.debug("No disributed virtual switch found with name {}".format(network_name))
5708
5709         except Exception as exp:
5710             self.logger.error("Error occurred while creating disributed virtaul port group {}"\
5711                              " : {}".format(network_name, exp))
5712         return None
5713
5714     def reconfig_portgroup(self, content, dvPort_group_name , config_info={}):
5715         """
5716          Method to reconfigure disributed virtual portgroup
5717
5718             Args:
5719                 dvPort_group_name - name of disributed virtual portgroup
5720                 content - vCenter content object
5721                 config_info - disributed virtual portgroup configuration
5722
5723             Returns:
5724                 task object
5725         """
5726         try:
5727             dvPort_group = self.get_dvport_group(dvPort_group_name)
5728             if dvPort_group:
5729                 dv_pg_spec = vim.dvs.DistributedVirtualPortgroup.ConfigSpec()
5730                 dv_pg_spec.configVersion = dvPort_group.config.configVersion
5731                 dv_pg_spec.defaultPortConfig = vim.dvs.VmwareDistributedVirtualSwitch.VmwarePortConfigPolicy()
5732                 if "vlanID" in config_info:
5733                     dv_pg_spec.defaultPortConfig.vlan = vim.dvs.VmwareDistributedVirtualSwitch.VlanIdSpec()
5734                     dv_pg_spec.defaultPortConfig.vlan.vlanId = config_info.get('vlanID')
5735
5736                 task = dvPort_group.ReconfigureDVPortgroup_Task(spec=dv_pg_spec)
5737                 return task
5738             else:
5739                 return None
5740         except Exception as exp:
5741             self.logger.error("Error occurred while reconfiguraing disributed virtaul port group {}"\
5742                              " : {}".format(dvPort_group_name, exp))
5743             return None
5744
5745
5746     def destroy_dvport_group(self , dvPort_group_name):
5747         """
5748          Method to destroy disributed virtual portgroup
5749
5750             Args:
5751                 network_name - name of network/portgroup
5752
5753             Returns:
5754                 True if portgroup successfully got deleted else false
5755         """
5756         vcenter_conect, content = self.get_vcenter_content()
5757         try:
5758             status = None
5759             dvPort_group = self.get_dvport_group(dvPort_group_name)
5760             if dvPort_group:
5761                 task = dvPort_group.Destroy_Task()
5762                 status = self.wait_for_vcenter_task(task, vcenter_conect)
5763             return status
5764         except vmodl.MethodFault as exp:
5765             self.logger.error("Caught vmodl fault {} while deleting disributed virtaul port group {}".format(
5766                                                                     exp, dvPort_group_name))
5767             return None
5768
5769
5770     def get_dvport_group(self, dvPort_group_name):
5771         """
5772         Method to get disributed virtual portgroup
5773
5774             Args:
5775                 network_name - name of network/portgroup
5776
5777             Returns:
5778                 portgroup object
5779         """
5780         vcenter_conect, content = self.get_vcenter_content()
5781         dvPort_group = None
5782         try:
5783             container = content.viewManager.CreateContainerView(content.rootFolder, [vim.dvs.DistributedVirtualPortgroup], True)
5784             for item in container.view:
5785                 if item.key == dvPort_group_name:
5786                     dvPort_group = item
5787                     break
5788             return dvPort_group
5789         except vmodl.MethodFault as exp:
5790             self.logger.error("Caught vmodl fault {} for disributed virtaul port group {}".format(
5791                                                                             exp, dvPort_group_name))
5792             return None
5793
5794     def get_vlanID_from_dvs_portgr(self, dvPort_group_name):
5795         """
5796          Method to get disributed virtual portgroup vlanID
5797
5798             Args:
5799                 network_name - name of network/portgroup
5800
5801             Returns:
5802                 vlan ID
5803         """
5804         vlanId = None
5805         try:
5806             dvPort_group = self.get_dvport_group(dvPort_group_name)
5807             if dvPort_group:
5808                 vlanId = dvPort_group.config.defaultPortConfig.vlan.vlanId
5809         except vmodl.MethodFault as exp:
5810             self.logger.error("Caught vmodl fault {} for disributed virtaul port group {}".format(
5811                                                                             exp, dvPort_group_name))
5812         return vlanId
5813
5814
5815     def configure_vlanID(self, content, vcenter_conect, dvPort_group_name):
5816         """
5817          Method to configure vlanID in disributed virtual portgroup vlanID
5818
5819             Args:
5820                 network_name - name of network/portgroup
5821
5822             Returns:
5823                 None
5824         """
5825         vlanID = self.get_vlanID_from_dvs_portgr(dvPort_group_name)
5826         if vlanID == 0:
5827             #configure vlanID
5828             vlanID = self.genrate_vlanID(dvPort_group_name)
5829             config = {"vlanID":vlanID}
5830             task = self.reconfig_portgroup(content, dvPort_group_name,
5831                                     config_info=config)
5832             if task:
5833                 status= self.wait_for_vcenter_task(task, vcenter_conect)
5834                 if status:
5835                     self.logger.info("Reconfigured Port group {} for vlan ID {}".format(
5836                                                         dvPort_group_name,vlanID))
5837             else:
5838                 self.logger.error("Fail reconfigure portgroup {} for vlanID{}".format(
5839                                         dvPort_group_name, vlanID))
5840
5841
5842     def genrate_vlanID(self, network_name):
5843         """
5844          Method to get unused vlanID
5845             Args:
5846                 network_name - name of network/portgroup
5847             Returns:
5848                 vlanID
5849         """
5850         vlan_id = None
5851         used_ids = []
5852         if self.config.get('vlanID_range') == None:
5853             raise vimconn.vimconnConflictException("You must provide a 'vlanID_range' "\
5854                         "at config value before creating sriov network with vlan tag")
5855         if "used_vlanIDs" not in self.persistent_info:
5856                 self.persistent_info["used_vlanIDs"] = {}
5857         else:
5858             used_ids = self.persistent_info["used_vlanIDs"].values()
5859             #For python3
5860             #used_ids = list(self.persistent_info["used_vlanIDs"].values())
5861
5862         for vlanID_range in self.config.get('vlanID_range'):
5863             start_vlanid , end_vlanid = vlanID_range.split("-")
5864             if start_vlanid > end_vlanid:
5865                 raise vimconn.vimconnConflictException("Invalid vlan ID range {}".format(
5866                                                                         vlanID_range))
5867
5868             for id in xrange(int(start_vlanid), int(end_vlanid) + 1):
5869             #For python3
5870             #for id in range(int(start_vlanid), int(end_vlanid) + 1):
5871                 if id not in used_ids:
5872                     vlan_id = id
5873                     self.persistent_info["used_vlanIDs"][network_name] = vlan_id
5874                     return vlan_id
5875         if vlan_id is None:
5876             raise vimconn.vimconnConflictException("All Vlan IDs are in use")
5877
5878
5879     def get_obj(self, content, vimtype, name):
5880         """
5881          Get the vsphere object associated with a given text name
5882         """
5883         obj = None
5884         container = content.viewManager.CreateContainerView(content.rootFolder, vimtype, True)
5885         for item in container.view:
5886             if item.name == name:
5887                 obj = item
5888                 break
5889         return obj
5890
5891
5892     def insert_media_to_vm(self, vapp, image_id):
5893         """
5894         Method to insert media CD-ROM (ISO image) from catalog to vm.
5895         vapp - vapp object to get vm id
5896         Image_id - image id for cdrom to be inerted to vm
5897         """
5898         # create connection object
5899         vca = self.connect()
5900         try:
5901             # fetching catalog details
5902             rest_url = "{}/api/catalog/{}".format(self.url, image_id)
5903             if vca._session:
5904                 headers = {'Accept':'application/*+xml;version=' + API_VERSION,
5905                            'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
5906                 response = self.perform_request(req_type='GET',
5907                                                 url=rest_url,
5908                                                 headers=headers)
5909
5910             if response.status_code != 200:
5911                 self.logger.error("REST call {} failed reason : {}"\
5912                              "status code : {}".format(url_rest_call,
5913                                                     response.content,
5914                                                response.status_code))
5915                 raise vimconn.vimconnException("insert_media_to_vm(): Failed to get "\
5916                                                                     "catalog details")
5917             # searching iso name and id
5918             iso_name,media_id = self.get_media_details(vca, response.content)
5919
5920             if iso_name and media_id:
5921                 data ="""<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
5922                      <ns6:MediaInsertOrEjectParams
5923                      xmlns="http://www.vmware.com/vcloud/versions" xmlns:ns2="http://schemas.dmtf.org/ovf/envelope/1" xmlns:ns3="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData" xmlns:ns4="http://schemas.dmtf.org/wbem/wscim/1/common" xmlns:ns5="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData" xmlns:ns6="http://www.vmware.com/vcloud/v1.5" xmlns:ns7="http://www.vmware.com/schema/ovf" xmlns:ns8="http://schemas.dmtf.org/ovf/environment/1" xmlns:ns9="http://www.vmware.com/vcloud/extension/v1.5">
5924                      <ns6:Media
5925                         type="application/vnd.vmware.vcloud.media+xml"
5926                         name="{}.iso"
5927                         id="urn:vcloud:media:{}"
5928                         href="https://{}/api/media/{}"/>
5929                      </ns6:MediaInsertOrEjectParams>""".format(iso_name, media_id,
5930                                                                 self.url,media_id)
5931
5932                 for vms in vapp.get_all_vms():
5933                     vm_id = vms.get('id').split(':')[-1]
5934
5935                     headers['Content-Type'] = 'application/vnd.vmware.vcloud.mediaInsertOrEjectParams+xml'
5936                     rest_url = "{}/api/vApp/vm-{}/media/action/insertMedia".format(self.url,vm_id)
5937
5938                     response = self.perform_request(req_type='POST',
5939                                                        url=rest_url,
5940                                                           data=data,
5941                                                     headers=headers)
5942
5943                     if response.status_code != 202:
5944                         self.logger.error("Failed to insert CD-ROM to vm")
5945                         raise vimconn.vimconnException("insert_media_to_vm() : Failed to insert"\
5946                                                                                     "ISO image to vm")
5947                     else:
5948                         task = self.get_task_from_response(response.content)
5949                         result = self.client.get_task_monitor().wait_for_success(task=task)
5950                         if result.get('status') == 'success':
5951                             self.logger.info("insert_media_to_vm(): Sucessfully inserted media ISO"\
5952                                                                     " image to vm {}".format(vm_id))
5953
5954         except Exception as exp:
5955             self.logger.error("insert_media_to_vm() : exception occurred "\
5956                                             "while inserting media CD-ROM")
5957             raise vimconn.vimconnException(message=exp)
5958
5959
5960     def get_media_details(self, vca, content):
5961         """
5962         Method to get catalog item details
5963         vca - connection object
5964         content - Catalog details
5965         Return - Media name, media id
5966         """
5967         cataloghref_list = []
5968         try:
5969             if content:
5970                 vm_list_xmlroot = XmlElementTree.fromstring(content)
5971                 for child in vm_list_xmlroot.iter():
5972                     if 'CatalogItem' in child.tag:
5973                         cataloghref_list.append(child.attrib.get('href'))
5974                 if cataloghref_list is not None:
5975                     for href in cataloghref_list:
5976                         if href:
5977                             headers = {'Accept':'application/*+xml;version=' + API_VERSION,
5978                            'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
5979                             response = self.perform_request(req_type='GET',
5980                                                                   url=href,
5981                                                            headers=headers)
5982                             if response.status_code != 200:
5983                                 self.logger.error("REST call {} failed reason : {}"\
5984                                              "status code : {}".format(href,
5985                                                            response.content,
5986                                                       response.status_code))
5987                                 raise vimconn.vimconnException("get_media_details : Failed to get "\
5988                                                                          "catalogitem details")
5989                             list_xmlroot = XmlElementTree.fromstring(response.content)
5990                             for child in list_xmlroot.iter():
5991                                 if 'Entity' in child.tag:
5992                                     if 'media' in child.attrib.get('href'):
5993                                         name = child.attrib.get('name')
5994                                         media_id = child.attrib.get('href').split('/').pop()
5995                                         return name,media_id
5996                             else:
5997                                 self.logger.debug("Media name and id not found")
5998                                 return False,False
5999         except Exception as exp:
6000             self.logger.error("get_media_details : exception occurred "\
6001                                                "getting media details")
6002             raise vimconn.vimconnException(message=exp)
6003
6004
6005     def retry_rest(self, method, url, add_headers=None, data=None):
6006         """ Method to get Token & retry respective REST request
6007             Args:
6008                 api - REST API - Can be one of 'GET' or 'PUT' or 'POST'
6009                 url - request url to be used
6010                 add_headers - Additional headers (optional)
6011                 data - Request payload data to be passed in request
6012             Returns:
6013                 response - Response of request
6014         """
6015         response = None
6016
6017         #Get token
6018         self.get_token()
6019
6020         if self.client._session:
6021                 headers = {'Accept':'application/*+xml;version=' + API_VERSION,
6022                            'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
6023
6024         if add_headers:
6025             headers.update(add_headers)
6026
6027         if method == 'GET':
6028             response = self.perform_request(req_type='GET',
6029                                             url=url,
6030                                             headers=headers)
6031         elif method == 'PUT':
6032             response = self.perform_request(req_type='PUT',
6033                                             url=url,
6034                                             headers=headers,
6035                                             data=data)
6036         elif method == 'POST':
6037             response = self.perform_request(req_type='POST',
6038                                             url=url,
6039                                             headers=headers,
6040                                             data=data)
6041         elif method == 'DELETE':
6042             response = self.perform_request(req_type='DELETE',
6043                                             url=url,
6044                                             headers=headers)
6045         return response
6046
6047
6048     def get_token(self):
6049         """ Generate a new token if expired
6050
6051             Returns:
6052                 The return client object that letter can be used to connect to vCloud director as admin for VDC
6053         """
6054         try:
6055             self.logger.debug("Generate token for vca {} as {} to datacenter {}.".format(self.org_name,
6056                                                                                       self.user,
6057                                                                                       self.org_name))
6058             host = self.url
6059             client = Client(host, verify_ssl_certs=False)
6060             client.set_credentials(BasicLoginCredentials(self.user, self.org_name, self.passwd))
6061             # connection object
6062             self.client = client
6063
6064         except:
6065             raise vimconn.vimconnConnectionException("Can't connect to a vCloud director org: "
6066                                                      "{} as user: {}".format(self.org_name, self.user))
6067
6068         if not client:
6069             raise vimconn.vimconnConnectionException("Failed while reconnecting vCD")
6070
6071
6072     def get_vdc_details(self):
6073         """ Get VDC details using pyVcloud Lib
6074
6075             Returns org and vdc object
6076         """
6077         org = Org(self.client, resource=self.client.get_org())
6078         vdc = org.get_vdc(self.tenant_name)
6079
6080         #Retry once, if failed by refreshing token
6081         if vdc is None:
6082             self.get_token()
6083             vdc = org.get_vdc(self.tenant_name)
6084
6085         return org, vdc
6086
6087
6088     def perform_request(self, req_type, url, headers=None, data=None):
6089         """Perform the POST/PUT/GET/DELETE request."""
6090
6091         #Log REST request details
6092         self.log_request(req_type, url=url, headers=headers, data=data)
6093         # perform request and return its result
6094         if req_type == 'GET':
6095             response = requests.get(url=url,
6096                                 headers=headers,
6097                                 verify=False)
6098         elif req_type == 'PUT':
6099             response = requests.put(url=url,
6100                                 headers=headers,
6101                                 data=data,
6102                                 verify=False)
6103         elif req_type == 'POST':
6104             response = requests.post(url=url,
6105                                  headers=headers,
6106                                  data=data,
6107                                  verify=False)
6108         elif req_type == 'DELETE':
6109             response = requests.delete(url=url,
6110                                  headers=headers,
6111                                  verify=False)
6112         #Log the REST response
6113         self.log_response(response)
6114
6115         return response
6116
6117
6118     def log_request(self, req_type, url=None, headers=None, data=None):
6119         """Logs REST request details"""
6120
6121         if req_type is not None:
6122             self.logger.debug("Request type: {}".format(req_type))
6123
6124         if url is not None:
6125             self.logger.debug("Request url: {}".format(url))
6126
6127         if headers is not None:
6128             for header in headers:
6129                 self.logger.debug("Request header: {}: {}".format(header, headers[header]))
6130
6131         if data is not None:
6132             self.logger.debug("Request data: {}".format(data))
6133
6134
6135     def log_response(self, response):
6136         """Logs REST response details"""
6137
6138         self.logger.debug("Response status code: {} ".format(response.status_code))
6139
6140
6141     def get_task_from_response(self, content):
6142         """
6143         content - API response content(response.content)
6144         return task object
6145         """
6146         xmlroot = XmlElementTree.fromstring(content)
6147         if xmlroot.tag.split('}')[1] == "Task":
6148             return xmlroot
6149         else:
6150             for ele in xmlroot:
6151                 if ele.tag.split("}")[1] == "Tasks":
6152                     task = ele[0]
6153                     break
6154             return task
6155
6156
6157     def power_on_vapp(self,vapp_id, vapp_name):
6158         """
6159         vapp_id - vApp uuid
6160         vapp_name - vAapp name
6161         return - Task object
6162         """
6163         headers = {'Accept':'application/*+xml;version=' + API_VERSION,
6164                    'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']}
6165
6166         poweron_href = "{}/api/vApp/vapp-{}/power/action/powerOn".format(self.url,
6167                                                                           vapp_id)
6168         response = self.perform_request(req_type='POST',
6169                                        url=poweron_href,
6170                                         headers=headers)
6171
6172         if response.status_code != 202:
6173             self.logger.error("REST call {} failed reason : {}"\
6174                          "status code : {} ".format(poweron_href,
6175                                                 response.content,
6176                                            response.status_code))
6177             raise vimconn.vimconnException("power_on_vapp() : Failed to power on "\
6178                                                       "vApp {}".format(vapp_name))
6179         else:
6180             poweron_task = self.get_task_from_response(response.content)
6181             return poweron_task
6182
6183