00ef9ef2c4673f6dd93bbc845b80d0245247df5a
1 # -*- coding: utf-8 -*-
4 # Copyright 2016-2019 VMware Inc.
5 # This file is part of ETSI OSM
8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
9 # not use this file except in compliance with the License. You may obtain
10 # a copy of the License at
12 # http://www.apache.org/licenses/LICENSE-2.0
14 # Unless required by applicable law or agreed to in writing, software
15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17 # License for the specific language governing permissions and limitations
20 # For those usages not covered by the Apache License, Version 2.0 please
21 # contact: osslegalrouting@vmware.com
28 from xml
.etree
import ElementTree
as XmlElementTree
31 from pyvcloud
.vcd
.client
import BasicLoginCredentials
32 from pyvcloud
.vcd
.client
import Client
34 from osm_mon
.collector
.utils
import CollectorUtils
35 from osm_mon
.collector
.vnf_collectors
.base_vim
import BaseVimCollector
36 from osm_mon
.collector
.vnf_metric
import VnfMetric
37 from osm_mon
.core
.common_db
import CommonDbClient
38 from osm_mon
.core
.config
import Config
40 log
= logging
.getLogger(__name__
)
46 # Ref: https://docs.vmware.com/en/vRealize-Operations-Manager/7.0/vrealize-operations-manager-70-reference-guide.pdf
47 # Potential metrics of interest
48 # "cpu|capacity_contentionPct"
49 # "cpu|corecount_provisioned"
53 # "cpu|effective_limit"
58 # "cpu|usagemhz_average"
59 # "cpu|usagemhz_average_mtd"
60 # "cpu|vm_capacity_provisioned"
62 # "guestfilesystem|percentage_total"
63 # "guestfilesystem|usage_total"
66 # "mem|host_contentionPct"
67 # "mem|reservation_used"
68 # "mem|swapinRate_average"
69 # "mem|swapoutRate_average"
70 # "mem|swapped_average"
72 # "net:Aggregate of all instances|droppedPct"
73 # "net|broadcastTx_summation"
74 # "net|droppedTx_summation"
75 # "net|multicastTx_summation"
76 # "net|pnicBytesRx_average"
77 # "net|pnicBytesTx_average"
78 # "net|received_average"
79 # "net|transmitted_average"
81 # "virtualDisk:Aggregate of all instances|commandsAveraged_average"
82 # "virtualDisk:Aggregate of all instances|numberReadAveraged_average"
83 # "virtualDisk:Aggregate of all instances|numberWriteAveraged_average"
84 # "virtualDisk:Aggregate of all instances|totalLatency"
85 # "virtualDisk:Aggregate of all instances|totalReadLatency_average"
86 # "virtualDisk:Aggregate of all instances|totalWriteLatency_average"
87 # "virtualDisk:Aggregate of all instances|usage"
88 # "virtualDisk:Aggregate of all instances|vDiskOIO"
89 # "virtualDisk|read_average"
90 # "virtualDisk|write_average"
93 # Percent guest operating system active memory
94 "average_memory_utilization": "mem|usage_average",
95 # Percentage of CPU that was used out of all the CPU that was allocated
96 "cpu_utilization": "cpu|usage_average",
97 # KB/s of data read in the performance interval
98 "disk_read_bytes": "virtualDisk|read_average",
99 # Average of read commands per second during the collection interval.
100 "disk_read_ops": "virtualDisk:aggregate of all instances|numberReadAveraged_average",
101 # KB/s of data written in the performance interval
102 "disk_write_bytes": "virtualDisk|write_average",
103 # Average of write commands per second during the collection interval.
104 "disk_write_ops": "virtualDisk:aggregate of all instances|numberWriteAveraged_average",
105 # "packets_in_dropped": "net|droppedRx_summation", # Not supported by vROPS
106 # Transmitted packets dropped in the collection interval
107 "packets_out_dropped": "net|droppedTx_summation",
108 # Bytes received in the performance interval
109 "packets_received": "net|received_average",
110 # Packets transmitted in the performance interval
111 "packets_sent": "net|transmitted_average",
114 # If the unit from vROPS does not align with the expected value. multiply by the specified amount to ensure
115 # the correct unit is returned.
116 METRIC_MULTIPLIERS
= {
117 "disk_read_bytes": 1024,
118 "disk_write_bytes": 1024,
119 "net_bytes_received": 1024,
120 "net_bytes_sent": 1024
124 class VMwareCollector(BaseVimCollector
):
125 def __init__(self
, config
: Config
, vim_account_id
: str):
126 super().__init
__(config
, vim_account_id
)
127 self
.common_db
= CommonDbClient(config
)
128 vim_account
= self
.get_vim_account(vim_account_id
)
129 self
.vrops_site
= vim_account
['vrops_site']
130 self
.vrops_user
= vim_account
['vrops_user']
131 self
.vrops_password
= vim_account
['vrops_password']
132 self
.vcloud_site
= vim_account
['vim_url']
133 self
.admin_username
= vim_account
['admin_username']
134 self
.admin_password
= vim_account
['admin_password']
135 self
.vim_uuid
= vim_account
['vim_uuid']
137 def connect_as_admin(self
):
138 """ Method connect as pvdc admin user to vCloud director.
139 There are certain action that can be done only by provider vdc admin user.
140 Organization creation / provider network creation etc.
143 The return client object that letter can be used to connect to vcloud direct as admin for provider vdc
146 log
.info("Logging into vCD org as admin.")
150 host
= self
.vcloud_site
151 admin_user
= self
.admin_username
152 admin_passwd
= self
.admin_password
154 client
= Client(host
, verify_ssl_certs
=False)
155 client
.set_highest_supported_version()
156 client
.set_credentials(BasicLoginCredentials(admin_user
, org
,
160 except Exception as e
:
161 log
.info("Can't connect to a vCloud director as: {} with exception {}".format(admin_user
, e
))
163 def _get_resource_uuid(self
, nsr_id
, vnf_member_index
, vdur_name
) -> str:
164 vdur
= self
.common_db
.get_vdur(nsr_id
, vnf_member_index
, vdur_name
)
165 return vdur
['vim-id']
167 def get_vim_account(self
, vim_account_id
: str):
169 Method to get VIM account details by its ID
171 return - dict with vim account details
174 vim_account_info
= CollectorUtils
.get_credentials(vim_account_id
)
176 vim_account
['name'] = vim_account_info
.name
177 vim_account
['vim_tenant_name'] = vim_account_info
.tenant_name
178 vim_account
['vim_type'] = vim_account_info
.type
179 vim_account
['vim_url'] = vim_account_info
.url
180 vim_account
['org_user'] = vim_account_info
.user
181 vim_account
['org_password'] = vim_account_info
.password
182 vim_account
['vim_uuid'] = vim_account_info
.uuid
184 vim_config
= json
.loads(vim_account_info
.config
)
185 vim_account
['admin_username'] = vim_config
['admin_username']
186 vim_account
['admin_password'] = vim_config
['admin_password']
187 vim_account
['vrops_site'] = vim_config
['vrops_site']
188 vim_account
['vrops_user'] = vim_config
['vrops_user']
189 vim_account
['vrops_password'] = vim_config
['vrops_password']
190 vim_account
['vcenter_ip'] = vim_config
['vcenter_ip']
191 vim_account
['vcenter_port'] = vim_config
['vcenter_port']
192 vim_account
['vcenter_user'] = vim_config
['vcenter_user']
193 vim_account
['vcenter_password'] = vim_config
['vcenter_password']
195 if vim_config
['nsx_manager'] is not None:
196 vim_account
['nsx_manager'] = vim_config
['nsx_manager']
198 if vim_config
['nsx_user'] is not None:
199 vim_account
['nsx_user'] = vim_config
['nsx_user']
201 if vim_config
['nsx_password'] is not None:
202 vim_account
['nsx_password'] = vim_config
['nsx_password']
204 if vim_config
['orgname'] is not None:
205 vim_account
['orgname'] = vim_config
['orgname']
209 def get_vm_moref_id(self
, vapp_uuid
):
211 Method to get the moref_id of given VM
218 vm_details
= self
.get_vapp_details_rest(vapp_uuid
)
220 if vm_details
and "vm_vcenter_info" in vm_details
:
221 vm_moref_id
= vm_details
["vm_vcenter_info"].get("vm_moref_id", None)
223 log
.info("Found vm_moref_id: {} for vApp UUID: {}".format(vm_moref_id
, vapp_uuid
))
226 except Exception as exp
:
227 log
.info("Error occurred while getting VM moref ID for VM : {}\n{}".format(exp
, traceback
.format_exc()))
229 def get_vapp_details_rest(self
, vapp_uuid
=None):
231 Method retrieve vapp detail from vCloud director
232 vapp_uuid - is vapp identifier.
233 Returns - VM MOref ID or return None
237 if vapp_uuid
is None:
238 return parsed_respond
240 vca
= self
.connect_as_admin()
243 log
.error("Failed to connect to vCD")
244 return parsed_respond
246 url_list
= [self
.vcloud_site
, '/api/vApp/vapp-', vapp_uuid
]
247 get_vapp_restcall
= ''.join(url_list
)
250 headers
= {'Accept': 'application/*+xml;version=' + API_VERSION
,
251 'x-vcloud-authorization': vca
._session
.headers
['x-vcloud-authorization']}
252 response
= requests
.get(get_vapp_restcall
,
256 if response
.status_code
!= 200:
257 log
.error("REST API call {} failed. Return status code {}".format(get_vapp_restcall
,
259 return parsed_respond
262 xmlroot_respond
= XmlElementTree
.fromstring(response
.content
)
264 namespaces
= {'vm': 'http://www.vmware.com/vcloud/v1.5',
265 "vmext": "http://www.vmware.com/vcloud/extension/v1.5",
266 "xmlns": "http://www.vmware.com/vcloud/v1.5"}
268 # parse children section for other attrib
269 children_section
= xmlroot_respond
.find('vm:Children/', namespaces
)
270 if children_section
is not None:
271 vCloud_extension_section
= children_section
.find('xmlns:VCloudExtension', namespaces
)
272 if vCloud_extension_section
is not None:
274 vim_info
= vCloud_extension_section
.find('vmext:VmVimInfo', namespaces
)
275 vmext
= vim_info
.find('vmext:VmVimObjectRef', namespaces
)
276 if vmext
is not None:
277 vm_vcenter_info
["vm_moref_id"] = vmext
.find('vmext:MoRef', namespaces
).text
278 parsed_respond
["vm_vcenter_info"] = vm_vcenter_info
280 except Exception as exp
:
281 log
.info("Error occurred for getting vApp details: {}\n{}".format(exp
,
282 traceback
.format_exc())
285 return parsed_respond
287 def get_vm_resource_id(self
, vm_moref_id
):
288 """ Find resource ID in vROPs using vm_moref_id
290 if vm_moref_id
is None:
293 api_url
= '/suite-api/api/resources?resourceKind=VirtualMachine'
294 headers
= {'Accept': 'application/json'}
296 resp
= requests
.get(self
.vrops_site
+ api_url
,
297 auth
=(self
.vrops_user
, self
.vrops_password
),
298 verify
=False, headers
=headers
)
300 if resp
.status_code
!= 200:
301 log
.error("Failed to get resource details for{} {} {}".format(vm_moref_id
,
306 vm_resource_id
= None
308 resp_data
= json
.loads(resp
.content
.decode('utf-8'))
309 if resp_data
.get('resourceList') is not None:
310 resource_list
= resp_data
.get('resourceList')
311 for resource
in resource_list
:
312 if resource
.get('resourceKey') is not None:
313 resource_details
= resource
['resourceKey']
314 if resource_details
.get('resourceIdentifiers') is not None:
315 resource_identifiers
= resource_details
['resourceIdentifiers']
316 for resource_identifier
in resource_identifiers
:
317 if resource_identifier
['identifierType']['name'] == 'VMEntityObjectID':
318 if resource_identifier
.get('value') is not None and \
319 resource_identifier
['value'] == vm_moref_id
:
320 vm_resource_id
= resource
['identifier']
321 log
.info("Found VM resource ID: {} for vm_moref_id: {}".format(vm_resource_id
,
324 except Exception as exp
:
325 log
.info("get_vm_resource_id: Error in parsing {}\n{}".format(exp
, traceback
.format_exc()))
327 return vm_resource_id
329 def collect(self
, vnfr
: dict):
330 nsr_id
= vnfr
['nsr-id-ref']
331 vnf_member_index
= vnfr
['member-vnf-index-ref']
332 vnfd
= self
.common_db
.get_vnfd(vnfr
['vnfd-id'])
334 for vdur
in vnfr
['vdur']:
335 # This avoids errors when vdur records have not been completely filled
336 if 'name' not in vdur
:
339 filter(lambda vdu
: vdu
['id'] == vdur
['vdu-id-ref'], vnfd
['vdu'])
343 if 'monitoring-param' not in vdu
:
346 resource_uuid
= self
._get
_resource
_uuid
(nsr_id
, vnf_member_index
, vdur
['name'])
348 # Find vm_moref_id from vApp uuid in vCD
349 vm_moref_id
= self
.get_vm_moref_id(resource_uuid
)
350 if vm_moref_id
is None:
351 log
.debug("Failed to find vm morefid for vApp in vCD: {}".format(resource_uuid
))
354 # Based on vm_moref_id, find VM's corresponding resource_id in vROPs
355 resource_id
= self
.get_vm_resource_id(vm_moref_id
)
356 if resource_id
is None:
357 log
.debug("Failed to find resource in vROPs: {}".format(resource_uuid
))
361 monitoring_params
= []
362 for metric_entry
in vdu
['monitoring-param']:
363 metric_name
= metric_entry
['nfvi-metric']
364 if metric_name
not in METRIC_MAPPINGS
:
365 log
.debug("Metric {} not supported, ignoring".format(metric_name
))
367 monitoring_params
.append(metric_name
)
368 vrops_metric_name
= METRIC_MAPPINGS
[metric_name
]
369 stat_key
= "{}&statKey={}".format(stat_key
, vrops_metric_name
)
372 end_time
= int(round(time
.time() * 1000))
373 begin_time
= end_time
- TEN_MINUTES
375 api_url
= "/suite-api/api/resources/stats?resourceId={}&begin={}&end={}{}".format(
376 resource_id
, str(begin_time
), str(end_time
), stat_key
)
377 headers
= {'Accept': 'application/json'}
379 resp
= requests
.get(self
.vrops_site
+ api_url
,
380 auth
=(self
.vrops_user
, self
.vrops_password
), verify
=False, headers
=headers
383 if resp
.status_code
!= 200:
384 log
.info("Failed to get Metrics data from vROPS for {} {} {}".format(vdur
.name
,
389 m_data
= json
.loads(resp
.content
.decode('utf-8'))
391 stat_list
= m_data
['values'][0]['stat-list']['stat']
392 for item
in stat_list
:
393 reported_metric
= item
['statKey']['key']
394 if reported_metric
not in METRIC_MAPPINGS
.values():
397 metric_name
= list(METRIC_MAPPINGS
.keys())[list(METRIC_MAPPINGS
.values()).
398 index(reported_metric
)]
399 if metric_name
in monitoring_params
:
400 metric_value
= item
['data'][-1]
401 if metric_name
in METRIC_MULTIPLIERS
:
402 metric_value
*= METRIC_MULTIPLIERS
[metric_name
]
403 metric
= VnfMetric(nsr_id
,
409 metrics
.append(metric
)
411 except Exception as e
:
412 log
.debug("No metric found for {}: %s".format(vdur
['name']), e
)