Reformat MON to standardized format
Change-Id: I5869a8c1d0a53c5f6ad6b8859e6469d447bfb63d
Signed-off-by: garciadeblas <gerardo.garciadeblas@telefonica.com>
diff --git a/osm_mon/collector/backends/prometheus.py b/osm_mon/collector/backends/prometheus.py
index fbe6d36..a9bb938 100644
--- a/osm_mon/collector/backends/prometheus.py
+++ b/osm_mon/collector/backends/prometheus.py
@@ -31,44 +31,44 @@
log = logging.getLogger(__name__)
-OSM_METRIC_PREFIX = 'osm_'
+OSM_METRIC_PREFIX = "osm_"
class PrometheusBackend(BaseBackend):
-
def __init__(self):
self.custom_collector = CustomCollector()
self._start_exporter(8000)
def handle(self, metrics: List[Metric]):
- log.debug('handle')
- log.debug('metrics: %s', metrics)
+ log.debug("handle")
+ log.debug("metrics: %s", metrics)
prometheus_metrics = {}
for metric in metrics:
if metric.name not in prometheus_metrics:
prometheus_metrics[metric.name] = GaugeMetricFamily(
OSM_METRIC_PREFIX + metric.name,
- 'OSM metric',
- labels=list(metric.tags.keys())
+ "OSM metric",
+ labels=list(metric.tags.keys()),
)
- prometheus_metrics[metric.name].add_metric(list(metric.tags.values()), metric.value)
+ prometheus_metrics[metric.name].add_metric(
+ list(metric.tags.values()), metric.value
+ )
self.custom_collector.metrics = prometheus_metrics.values()
def _start_exporter(self, port):
- log.debug('_start_exporter')
- log.debug('port: %s', port)
+ log.debug("_start_exporter")
+ log.debug("port: %s", port)
REGISTRY.register(self.custom_collector)
log.info("Starting MON Prometheus exporter at port %s", port)
start_http_server(port)
class CustomCollector(object):
-
def __init__(self):
self.metrics = []
def describe(self):
- log.debug('describe')
+ log.debug("describe")
return []
def collect(self):
diff --git a/osm_mon/collector/collector.py b/osm_mon/collector/collector.py
index ad166e4..a69e651 100644
--- a/osm_mon/collector/collector.py
+++ b/osm_mon/collector/collector.py
@@ -29,9 +29,7 @@
log = logging.getLogger(__name__)
-METRIC_BACKENDS = [
- PrometheusBackend
-]
+METRIC_BACKENDS = [PrometheusBackend]
class Collector:
@@ -42,11 +40,11 @@
self._init_backends()
def collect_forever(self):
- log.debug('collect_forever')
+ log.debug("collect_forever")
while True:
try:
self.collect_metrics()
- time.sleep(int(self.conf.get('collector', 'interval')))
+ time.sleep(int(self.conf.get("collector", "interval")))
except Exception:
log.exception("Error collecting metrics")
diff --git a/osm_mon/collector/infra_collectors/base.py b/osm_mon/collector/infra_collectors/base.py
index 586169a..2f97ebc 100644
--- a/osm_mon/collector/infra_collectors/base.py
+++ b/osm_mon/collector/infra_collectors/base.py
@@ -25,6 +25,5 @@
class BaseInfraCollector:
-
def collect(self) -> List[Metric]:
pass
diff --git a/osm_mon/collector/infra_collectors/base_osinfra.py b/osm_mon/collector/infra_collectors/base_osinfra.py
index 1c1999c..61e9e42 100644
--- a/osm_mon/collector/infra_collectors/base_osinfra.py
+++ b/osm_mon/collector/infra_collectors/base_osinfra.py
@@ -46,46 +46,46 @@
def collect(self) -> List[Metric]:
metrics = []
vim_status = self.is_vim_ok()
- if self.vim_account['_admin']['projects_read']:
- vim_project_id = self.vim_account['_admin']['projects_read'][0]
+ if self.vim_account["_admin"]["projects_read"]:
+ vim_project_id = self.vim_account["_admin"]["projects_read"][0]
else:
- vim_project_id = ''
+ vim_project_id = ""
vim_tags = {
- 'vim_account_id': self.vim_account['_id'],
- 'project_id': vim_project_id
+ "vim_account_id": self.vim_account["_id"],
+ "project_id": vim_project_id,
}
- vim_status_metric = Metric(vim_tags, 'vim_status', vim_status)
+ vim_status_metric = Metric(vim_tags, "vim_status", vim_status)
metrics.append(vim_status_metric)
- vnfrs = self.common_db.get_vnfrs(vim_account_id=self.vim_account['_id'])
+ vnfrs = self.common_db.get_vnfrs(vim_account_id=self.vim_account["_id"])
for vnfr in vnfrs:
- nsr_id = vnfr['nsr-id-ref']
- ns_name = self.common_db.get_nsr(nsr_id)['name']
- vnf_member_index = vnfr['member-vnf-index-ref']
- if vnfr['_admin']['projects_read']:
- vnfr_project_id = vnfr['_admin']['projects_read'][0]
+ nsr_id = vnfr["nsr-id-ref"]
+ ns_name = self.common_db.get_nsr(nsr_id)["name"]
+ vnf_member_index = vnfr["member-vnf-index-ref"]
+ if vnfr["_admin"]["projects_read"]:
+ vnfr_project_id = vnfr["_admin"]["projects_read"][0]
else:
- vnfr_project_id = ''
- for vdur in vnfr['vdur']:
- if 'vim-id' not in vdur:
+ vnfr_project_id = ""
+ for vdur in vnfr["vdur"]:
+ if "vim-id" not in vdur:
log.debug("Field vim-id is not present in vdur")
continue
- resource_uuid = vdur['vim-id']
+ resource_uuid = vdur["vim-id"]
tags = {
- 'vim_account_id': self.vim_account['_id'],
- 'resource_uuid': resource_uuid,
- 'nsr_id': nsr_id,
- 'ns_name': ns_name,
- 'vnf_member_index': vnf_member_index,
- 'vdur_name': vdur.get("name", ""),
- 'project_id': vnfr_project_id
+ "vim_account_id": self.vim_account["_id"],
+ "resource_uuid": resource_uuid,
+ "nsr_id": nsr_id,
+ "ns_name": ns_name,
+ "vnf_member_index": vnf_member_index,
+ "vdur_name": vdur.get("name", ""),
+ "project_id": vnfr_project_id,
}
try:
vm = self.nova.servers.get(resource_uuid)
- vm_status = (1 if vm.status == 'ACTIVE' else 0)
- vm_status_metric = Metric(tags, 'vm_status', vm_status)
+ vm_status = 1 if vm.status == "ACTIVE" else 0
+ vm_status_metric = Metric(tags, "vm_status", vm_status)
except Exception as e:
log.warning("VM status is not OK: %s" % e)
- vm_status_metric = Metric(tags, 'vm_status', 0)
+ vm_status_metric = Metric(tags, "vm_status", 0)
metrics.append(vm_status_metric)
return metrics
diff --git a/osm_mon/collector/infra_collectors/onos.py b/osm_mon/collector/infra_collectors/onos.py
index 33a3aa4..ccd66d5 100644
--- a/osm_mon/collector/infra_collectors/onos.py
+++ b/osm_mon/collector/infra_collectors/onos.py
@@ -42,29 +42,26 @@
def collect(self) -> List[Metric]:
metrics = []
sdnc_status = self.is_sdnc_ok()
- if self.sdnc['_admin']['projects_read']:
- sdnc_project_id = self.sdnc['_admin']['projects_read'][0]
+ if self.sdnc["_admin"]["projects_read"]:
+ sdnc_project_id = self.sdnc["_admin"]["projects_read"][0]
else:
- sdnc_project_id = ''
- sdnc_tags = {
- 'sdnc_id': self.sdnc['_id'],
- 'project_id': sdnc_project_id
- }
- sdnc_status_metric = Metric(sdnc_tags, 'sdnc_status', sdnc_status)
+ sdnc_project_id = ""
+ sdnc_tags = {"sdnc_id": self.sdnc["_id"], "project_id": sdnc_project_id}
+ sdnc_status_metric = Metric(sdnc_tags, "sdnc_status", sdnc_status)
metrics.append(sdnc_status_metric)
return metrics
def is_sdnc_ok(self) -> bool:
try:
- ip = self.sdnc['ip']
- port = self.sdnc['port']
- user = self.sdnc['user']
- password = self.common_db.decrypt_sdnc_password(self.sdnc['password'],
- self.sdnc['schema_version'],
- self.sdnc['_id'])
+ ip = self.sdnc["ip"]
+ port = self.sdnc["port"]
+ user = self.sdnc["user"]
+ password = self.common_db.decrypt_sdnc_password(
+ self.sdnc["password"], self.sdnc["schema_version"], self.sdnc["_id"]
+ )
# TODO: Add support for https
- url = 'http://{}:{}/onos/v1/devices'.format(ip, port)
+ url = "http://{}:{}/onos/v1/devices".format(ip, port)
requests.get(url, auth=HTTPBasicAuth(user, password))
return True
except Exception:
diff --git a/osm_mon/collector/infra_collectors/vmware.py b/osm_mon/collector/infra_collectors/vmware.py
index 89717dc..65e739d 100644
--- a/osm_mon/collector/infra_collectors/vmware.py
+++ b/osm_mon/collector/infra_collectors/vmware.py
@@ -37,30 +37,29 @@
from osm_mon.core.config import Config
log = logging.getLogger(__name__)
-API_VERSION = '27.0'
+API_VERSION = "27.0"
class VMwareInfraCollector(BaseVimInfraCollector):
-
def __init__(self, config: Config, vim_account_id: str):
super().__init__(config, vim_account_id)
self.vim_account_id = vim_account_id
self.common_db = CommonDbClient(config)
vim_account = self.get_vim_account(vim_account_id)
- self.vcloud_site = vim_account['vim_url']
- self.admin_username = vim_account['admin_username']
- self.admin_password = vim_account['admin_password']
- self.vim_uuid = vim_account['vim_uuid']
- self.org_name = vim_account['orgname']
- self.vim_project_id = vim_account['project_id']
+ self.vcloud_site = vim_account["vim_url"]
+ self.admin_username = vim_account["admin_username"]
+ self.admin_password = vim_account["admin_password"]
+ self.vim_uuid = vim_account["vim_uuid"]
+ self.org_name = vim_account["orgname"]
+ self.vim_project_id = vim_account["project_id"]
def connect_vim_as_admin(self):
- """ Method connect as pvdc admin user to vCloud director.
- There are certain action that can be done only by provider vdc admin user.
- Organization creation / provider network creation etc.
+ """Method connect as pvdc admin user to vCloud director.
+ There are certain action that can be done only by provider vdc admin user.
+ Organization creation / provider network creation etc.
- Returns:
- The return client object that letter can be used to connect to vcloud direct as admin for provider vdc
+ Returns:
+ The return client object that letter can be used to connect to vcloud direct as admin for provider vdc
"""
log.info("Logging into vCD org as admin.")
@@ -70,42 +69,45 @@
host = self.vcloud_site
admin_user = self.admin_username
admin_passwd = self.admin_password
- org = 'System'
+ org = "System"
client = Client(host, verify_ssl_certs=False)
client.set_highest_supported_version()
- client.set_credentials(BasicLoginCredentials(admin_user, org,
- admin_passwd))
+ client.set_credentials(BasicLoginCredentials(admin_user, org, admin_passwd))
return client
except Exception as e:
- log.info("Can't connect to a vCloud director as: {} with exception {}".format(admin_user, e))
+ log.info(
+ "Can't connect to a vCloud director as: {} with exception {}".format(
+ admin_user, e
+ )
+ )
def get_vim_account(self, vim_account_id: str):
"""
- Method to get VIM account details by its ID
- arg - VIM ID
- return - dict with vim account details
+ Method to get VIM account details by its ID
+ arg - VIM ID
+ return - dict with vim account details
"""
vim_account = {}
vim_account_info = self.common_db.get_vim_account(vim_account_id)
- vim_account['name'] = vim_account_info['name']
- vim_account['vim_tenant_name'] = vim_account_info['vim_tenant_name']
- vim_account['vim_type'] = vim_account_info['vim_type']
- vim_account['vim_url'] = vim_account_info['vim_url']
- vim_account['org_user'] = vim_account_info['vim_user']
- vim_account['vim_uuid'] = vim_account_info['_id']
- if vim_account_info['_admin']['projects_read']:
- vim_account['project_id'] = vim_account_info['_admin']['projects_read'][0]
+ vim_account["name"] = vim_account_info["name"]
+ vim_account["vim_tenant_name"] = vim_account_info["vim_tenant_name"]
+ vim_account["vim_type"] = vim_account_info["vim_type"]
+ vim_account["vim_url"] = vim_account_info["vim_url"]
+ vim_account["org_user"] = vim_account_info["vim_user"]
+ vim_account["vim_uuid"] = vim_account_info["_id"]
+ if vim_account_info["_admin"]["projects_read"]:
+ vim_account["project_id"] = vim_account_info["_admin"]["projects_read"][0]
else:
- vim_account['project_id'] = ''
+ vim_account["project_id"] = ""
- vim_config = vim_account_info['config']
- vim_account['admin_username'] = vim_config['admin_username']
- vim_account['admin_password'] = vim_config['admin_password']
+ vim_config = vim_account_info["config"]
+ vim_account["admin_username"] = vim_config["admin_username"]
+ vim_account["admin_password"] = vim_config["admin_password"]
- if vim_config['orgname'] is not None:
- vim_account['orgname'] = vim_config['orgname']
+ if vim_config["orgname"] is not None:
+ vim_account["orgname"] = vim_config["orgname"]
return vim_account
@@ -115,27 +117,36 @@
if client._session:
org_list = client.get_org_list()
for org in org_list.Org:
- if org.get('name') == self.org_name:
- org_uuid = org.get('href').split('/')[-1]
+ if org.get("name") == self.org_name:
+ org_uuid = org.get("href").split("/")[-1]
- url = '{}/api/org/{}'.format(self.vcloud_site, org_uuid)
+ url = "{}/api/org/{}".format(self.vcloud_site, org_uuid)
- headers = {'Accept': 'application/*+xml;version=' + API_VERSION,
- 'x-vcloud-authorization': client._session.headers['x-vcloud-authorization']}
+ headers = {
+ "Accept": "application/*+xml;version=" + API_VERSION,
+ "x-vcloud-authorization": client._session.headers[
+ "x-vcloud-authorization"
+ ],
+ }
- response = requests.get(url=url,
- headers=headers,
- verify=False)
+ response = requests.get(url=url, headers=headers, verify=False)
- if response.status_code != requests.codes.ok: # pylint: disable=no-member
+ if (
+ response.status_code != requests.codes.ok
+ ): # pylint: disable=no-member
log.info("check_vim_status(): failed to get org details")
else:
org_details = XmlElementTree.fromstring(response.content)
vdc_list = {}
for child in org_details:
- if 'type' in child.attrib:
- if child.attrib['type'] == 'application/vnd.vmware.vcloud.vdc+xml':
- vdc_list[child.attrib['href'].split("/")[-1:][0]] = child.attrib['name']
+ if "type" in child.attrib:
+ if (
+ child.attrib["type"]
+ == "application/vnd.vmware.vcloud.vdc+xml"
+ ):
+ vdc_list[
+ child.attrib["href"].split("/")[-1:][0]
+ ] = child.attrib["name"]
if vdc_list:
return True
@@ -148,22 +159,26 @@
try:
client = self.connect_vim_as_admin()
if client._session:
- url = '{}/api/vApp/vapp-{}'.format(self.vcloud_site, vapp_id)
+ url = "{}/api/vApp/vapp-{}".format(self.vcloud_site, vapp_id)
- headers = {'Accept': 'application/*+xml;version=' + API_VERSION,
- 'x-vcloud-authorization': client._session.headers['x-vcloud-authorization']}
+ headers = {
+ "Accept": "application/*+xml;version=" + API_VERSION,
+ "x-vcloud-authorization": client._session.headers[
+ "x-vcloud-authorization"
+ ],
+ }
- response = requests.get(url=url,
- headers=headers,
- verify=False)
+ response = requests.get(url=url, headers=headers, verify=False)
- if response.status_code != requests.codes.ok: # pylint: disable=no-member
+ if (
+ response.status_code != requests.codes.ok
+ ): # pylint: disable=no-member
log.info("check_vm_status(): failed to get vApp details")
else:
vapp_details = XmlElementTree.fromstring(response.content)
vm_list = []
for child in vapp_details:
- if child.tag.split("}")[1] == 'Children':
+ if child.tag.split("}")[1] == "Children":
for item in child.getchildren():
vm_list.append(item.attrib)
return vm_list
@@ -175,42 +190,39 @@
vim_status = self.check_vim_status()
vim_account_id = self.vim_account_id
vim_project_id = self.vim_project_id
- vim_tags = {
- 'vim_account_id': vim_account_id,
- 'project_id': vim_project_id
- }
- vim_status_metric = Metric(vim_tags, 'vim_status', vim_status)
+ vim_tags = {"vim_account_id": vim_account_id, "project_id": vim_project_id}
+ vim_status_metric = Metric(vim_tags, "vim_status", vim_status)
metrics.append(vim_status_metric)
vnfrs = self.common_db.get_vnfrs(vim_account_id=vim_account_id)
for vnfr in vnfrs:
- nsr_id = vnfr['nsr-id-ref']
- ns_name = self.common_db.get_nsr(nsr_id)['name']
- vnf_member_index = vnfr['member-vnf-index-ref']
- if vnfr['_admin']['projects_read']:
- vnfr_project_id = vnfr['_admin']['projects_read'][0]
+ nsr_id = vnfr["nsr-id-ref"]
+ ns_name = self.common_db.get_nsr(nsr_id)["name"]
+ vnf_member_index = vnfr["member-vnf-index-ref"]
+ if vnfr["_admin"]["projects_read"]:
+ vnfr_project_id = vnfr["_admin"]["projects_read"][0]
else:
- vnfr_project_id = ''
- for vdur in vnfr['vdur']:
- resource_uuid = vdur['vim-id']
+ vnfr_project_id = ""
+ for vdur in vnfr["vdur"]:
+ resource_uuid = vdur["vim-id"]
tags = {
- 'vim_account_id': self.vim_account_id,
- 'resource_uuid': resource_uuid,
- 'nsr_id': nsr_id,
- 'ns_name': ns_name,
- 'vnf_member_index': vnf_member_index,
- 'vdur_name': vdur['name'],
- 'project_id': vnfr_project_id
+ "vim_account_id": self.vim_account_id,
+ "resource_uuid": resource_uuid,
+ "nsr_id": nsr_id,
+ "ns_name": ns_name,
+ "vnf_member_index": vnf_member_index,
+ "vdur_name": vdur["name"],
+ "project_id": vnfr_project_id,
}
try:
vm_list = self.check_vm_status(resource_uuid)
for vm in vm_list:
- if vm['status'] == '4' and vm['deployed'] == 'true':
+ if vm["status"] == "4" and vm["deployed"] == "true":
vm_status = 1
else:
vm_status = 0
- vm_status_metric = Metric(tags, 'vm_status', vm_status)
+ vm_status_metric = Metric(tags, "vm_status", vm_status)
except Exception:
log.exception("VM status is not OK!")
- vm_status_metric = Metric(tags, 'vm_status', 0)
+ vm_status_metric = Metric(tags, "vm_status", 0)
metrics.append(vm_status_metric)
return metrics
diff --git a/osm_mon/collector/service.py b/osm_mon/collector/service.py
index 5eb65a9..aa27083 100644
--- a/osm_mon/collector/service.py
+++ b/osm_mon/collector/service.py
@@ -45,21 +45,17 @@
VIM_COLLECTORS = {
"openstack": OpenstackCollector,
"vmware": VMwareCollector,
- "vio": VIOCollector
+ "vio": VIOCollector,
}
VIM_INFRA_COLLECTORS = {
"openstack": OpenstackInfraCollector,
"vmware": VMwareInfraCollector,
- "vio": VIOInfraCollector
+ "vio": VIOInfraCollector,
}
-SDN_INFRA_COLLECTORS = {
- "onosof": OnosInfraCollector,
- "onos_vpls": OnosInfraCollector
-}
+SDN_INFRA_COLLECTORS = {"onosof": OnosInfraCollector, "onos_vpls": OnosInfraCollector}
class CollectorService:
-
def __init__(self, config: Config):
self.conf = config
self.common_db = CommonDbClient(self.conf)
@@ -70,11 +66,11 @@
def _get_vim_type(conf: Config, vim_account_id: str) -> str:
common_db = CommonDbClient(conf)
vim_account = common_db.get_vim_account(vim_account_id)
- vim_type = vim_account['vim_type']
- if 'config' in vim_account and 'vim_type' in vim_account['config']:
- vim_type = vim_account['config']['vim_type'].lower()
- if vim_type == 'vio' and 'vrops_site' not in vim_account['config']:
- vim_type = 'openstack'
+ vim_type = vim_account["vim_type"]
+ if "config" in vim_account and "vim_type" in vim_account["config"]:
+ vim_type = vim_account["config"]["vim_type"].lower()
+ if vim_type == "vio" and "vrops_site" not in vim_account["config"]:
+ vim_type = "openstack"
return vim_type
@staticmethod
@@ -117,7 +113,7 @@
log.info("Collecting sdnc metrics")
metrics = []
common_db = CommonDbClient(conf)
- sdn_type = common_db.get_sdnc(sdnc_id)['type']
+ sdn_type = common_db.get_sdnc(sdnc_id)["type"]
if sdn_type in SDN_INFRA_COLLECTORS:
collector = SDN_INFRA_COLLECTORS[sdn_type](conf, sdnc_id)
metrics = collector.collect()
@@ -128,9 +124,9 @@
@staticmethod
def _stop_process_pool(executor):
- log.info('Shutting down process pool')
+ log.info("Shutting down process pool")
try:
- log.debug('Stopping residual processes in the process pool')
+ log.debug("Stopping residual processes in the process pool")
for pid, process in executor._processes.items():
if process.is_alive():
process.terminate()
@@ -140,11 +136,11 @@
try:
# Shutting down executor
- log.debug('Shutting down process pool executor')
+ log.debug("Shutting down process pool executor")
executor.shutdown()
except RuntimeError as e:
- log.info('RuntimeError in shutting down executer')
- log.debug('RuntimeError %s' % (e))
+ log.info("RuntimeError in shutting down executer")
+ log.debug("RuntimeError %s" % (e))
return
def collect_metrics(self) -> List[Metric]:
@@ -153,37 +149,72 @@
start_time = time.time()
# Starting executor pool with pool size process_pool_size. Default process_pool_size is 20
- with concurrent.futures.ProcessPoolExecutor(self.conf.get('collector', 'process_pool_size')) as executor:
- log.info('Started metric collector process pool with pool size %s' % (self.conf.get('collector',
- 'process_pool_size')))
+ with concurrent.futures.ProcessPoolExecutor(
+ self.conf.get("collector", "process_pool_size")
+ ) as executor:
+ log.info(
+ "Started metric collector process pool with pool size %s"
+ % (self.conf.get("collector", "process_pool_size"))
+ )
futures = []
for vnfr in vnfrs:
- nsr_id = vnfr['nsr-id-ref']
- vnf_member_index = vnfr['member-vnf-index-ref']
- vim_account_id = self.common_db.get_vim_account_id(nsr_id, vnf_member_index)
- futures.append(executor.submit(CollectorService._collect_vim_metrics, self.conf, vnfr, vim_account_id))
- futures.append(executor.submit(CollectorService._collect_vca_metrics, self.conf, vnfr))
+ nsr_id = vnfr["nsr-id-ref"]
+ vnf_member_index = vnfr["member-vnf-index-ref"]
+ vim_account_id = self.common_db.get_vim_account_id(
+ nsr_id, vnf_member_index
+ )
+ futures.append(
+ executor.submit(
+ CollectorService._collect_vim_metrics,
+ self.conf,
+ vnfr,
+ vim_account_id,
+ )
+ )
+ futures.append(
+ executor.submit(
+ CollectorService._collect_vca_metrics, self.conf, vnfr
+ )
+ )
vims = self.common_db.get_vim_accounts()
for vim in vims:
- futures.append(executor.submit(CollectorService._collect_vim_infra_metrics, self.conf, vim['_id']))
+ futures.append(
+ executor.submit(
+ CollectorService._collect_vim_infra_metrics,
+ self.conf,
+ vim["_id"],
+ )
+ )
sdncs = self.common_db.get_sdncs()
for sdnc in sdncs:
- futures.append(executor.submit(CollectorService._collect_sdnc_infra_metrics, self.conf, sdnc['_id']))
+ futures.append(
+ executor.submit(
+ CollectorService._collect_sdnc_infra_metrics,
+ self.conf,
+ sdnc["_id"],
+ )
+ )
try:
# Wait for future calls to complete till process_execution_timeout. Default is 50 seconds
- for future in concurrent.futures.as_completed(futures, self.conf.get('collector',
- 'process_execution_timeout')):
- result = future.result(timeout=int(self.conf.get('collector',
- 'process_execution_timeout')))
+ for future in concurrent.futures.as_completed(
+ futures, self.conf.get("collector", "process_execution_timeout")
+ ):
+ result = future.result(
+ timeout=int(
+ self.conf.get("collector", "process_execution_timeout")
+ )
+ )
metrics.extend(result)
- log.debug('result = %s' % (result))
+ log.debug("result = %s" % (result))
except concurrent.futures.TimeoutError as e:
# Some processes have not completed due to timeout error
- log.info('Some processes have not finished due to TimeoutError exception')
- log.debug('concurrent.futures.TimeoutError exception %s' % (e))
+ log.info(
+ "Some processes have not finished due to TimeoutError exception"
+ )
+ log.debug("concurrent.futures.TimeoutError exception %s" % (e))
# Shutting down process pool executor
CollectorService._stop_process_pool(executor)
diff --git a/osm_mon/collector/utils/openstack.py b/osm_mon/collector/utils/openstack.py
index 09c472c..9162f98 100644
--- a/osm_mon/collector/utils/openstack.py
+++ b/osm_mon/collector/utils/openstack.py
@@ -27,26 +27,27 @@
class OpenstackUtils:
-
@staticmethod
def get_session(creds: dict):
verify_ssl = True
- project_domain_name = 'Default'
- user_domain_name = 'Default'
- if 'config' in creds:
- vim_config = creds['config']
- if 'insecure' in vim_config and vim_config['insecure']:
+ project_domain_name = "Default"
+ user_domain_name = "Default"
+ if "config" in creds:
+ vim_config = creds["config"]
+ if "insecure" in vim_config and vim_config["insecure"]:
verify_ssl = False
- if 'ca_cert' in vim_config:
- verify_ssl = vim_config['ca_cert']
- if 'project_domain_name' in vim_config:
- project_domain_name = vim_config['project_domain_name']
- if 'user_domain_name' in vim_config:
- user_domain_name = vim_config['user_domain_name']
- auth = v3.Password(auth_url=creds['vim_url'],
- username=creds['vim_user'],
- password=creds['vim_password'],
- project_name=creds['vim_tenant_name'],
- project_domain_name=project_domain_name,
- user_domain_name=user_domain_name)
+ if "ca_cert" in vim_config:
+ verify_ssl = vim_config["ca_cert"]
+ if "project_domain_name" in vim_config:
+ project_domain_name = vim_config["project_domain_name"]
+ if "user_domain_name" in vim_config:
+ user_domain_name = vim_config["user_domain_name"]
+ auth = v3.Password(
+ auth_url=creds["vim_url"],
+ username=creds["vim_user"],
+ password=creds["vim_password"],
+ project_name=creds["vim_tenant_name"],
+ project_domain_name=project_domain_name,
+ user_domain_name=user_domain_name,
+ )
return session.Session(auth=auth, verify=verify_ssl, timeout=10)
diff --git a/osm_mon/collector/vnf_collectors/openstack.py b/osm_mon/collector/vnf_collectors/openstack.py
index a5c4980..525bd00 100644
--- a/osm_mon/collector/vnf_collectors/openstack.py
+++ b/osm_mon/collector/vnf_collectors/openstack.py
@@ -55,21 +55,22 @@
"cpu_utilization": "cpu",
}
-METRIC_MULTIPLIERS = {
- "cpu": 0.0000001
-}
+METRIC_MULTIPLIERS = {"cpu": 0.0000001}
-METRIC_AGGREGATORS = {
- "cpu": "rate:mean"
-}
+METRIC_AGGREGATORS = {"cpu": "rate:mean"}
-INTERFACE_METRICS = ['packets_in_dropped', 'packets_out_dropped', 'packets_received', 'packets_sent']
+INTERFACE_METRICS = [
+ "packets_in_dropped",
+ "packets_out_dropped",
+ "packets_received",
+ "packets_sent",
+]
class MetricType(Enum):
- INSTANCE = 'instance'
- INTERFACE_ALL = 'interface_all'
- INTERFACE_ONE = 'interface_one'
+ INSTANCE = "instance"
+ INTERFACE_ALL = "interface_all"
+ INTERFACE_ONE = "interface_one"
class OpenstackCollector(BaseVimCollector):
@@ -83,59 +84,77 @@
sess = OpenstackUtils.get_session(vim_account)
return keystone_client.Client(session=sess)
- def _get_resource_uuid(self, nsr_id: str, vnf_member_index: str, vdur_name: str) -> str:
+ def _get_resource_uuid(
+ self, nsr_id: str, vnf_member_index: str, vdur_name: str
+ ) -> str:
vdur = self.common_db.get_vdur(nsr_id, vnf_member_index, vdur_name)
- return vdur['vim-id']
+ return vdur["vim-id"]
def collect(self, vnfr: dict) -> List[Metric]:
- nsr_id = vnfr['nsr-id-ref']
- vnf_member_index = vnfr['member-vnf-index-ref']
- vnfd = self.common_db.get_vnfd(vnfr['vnfd-id'])
+ nsr_id = vnfr["nsr-id-ref"]
+ vnf_member_index = vnfr["member-vnf-index-ref"]
+ vnfd = self.common_db.get_vnfd(vnfr["vnfd-id"])
# Populate extra tags for metrics
tags = {}
- tags['ns_name'] = self.common_db.get_nsr(nsr_id)['name']
- if vnfr['_admin']['projects_read']:
- tags['project_id'] = vnfr['_admin']['projects_read'][0]
+ tags["ns_name"] = self.common_db.get_nsr(nsr_id)["name"]
+ if vnfr["_admin"]["projects_read"]:
+ tags["project_id"] = vnfr["_admin"]["projects_read"][0]
else:
- tags['project_id'] = ''
+ tags["project_id"] = ""
metrics = []
- for vdur in vnfr['vdur']:
+ for vdur in vnfr["vdur"]:
# This avoids errors when vdur records have not been completely filled
- if 'name' not in vdur:
+ if "name" not in vdur:
continue
- vdu = next(
- filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu'])
- )
- if 'monitoring-parameter' in vdu:
- for param in vdu['monitoring-parameter']:
- metric_name = param['performance-metric']
+ vdu = next(filter(lambda vdu: vdu["id"] == vdur["vdu-id-ref"], vnfd["vdu"]))
+ if "monitoring-parameter" in vdu:
+ for param in vdu["monitoring-parameter"]:
+ metric_name = param["performance-metric"]
openstack_metric_name = METRIC_MAPPINGS[metric_name]
metric_type = self._get_metric_type(metric_name)
try:
- resource_id = self._get_resource_uuid(nsr_id, vnf_member_index, vdur['name'])
+ resource_id = self._get_resource_uuid(
+ nsr_id, vnf_member_index, vdur["name"]
+ )
except ValueError:
log.warning(
"Could not find resource_uuid for vdur %s, vnf_member_index %s, nsr_id %s. "
"Was it recently deleted?",
- vdur['name'], vnf_member_index, nsr_id)
+ vdur["name"],
+ vnf_member_index,
+ nsr_id,
+ )
continue
try:
log.info(
"Collecting metric type: %s and metric_name: %s and resource_id %s and ",
metric_type,
metric_name,
- resource_id)
- value = self.backend.collect_metric(metric_type, openstack_metric_name, resource_id)
+ resource_id,
+ )
+ value = self.backend.collect_metric(
+ metric_type, openstack_metric_name, resource_id
+ )
if value is not None:
log.info("value: %s", value)
- metric = VnfMetric(nsr_id, vnf_member_index, vdur['name'], metric_name, value, tags)
+ metric = VnfMetric(
+ nsr_id,
+ vnf_member_index,
+ vdur["name"],
+ metric_name,
+ value,
+ tags,
+ )
metrics.append(metric)
else:
log.info("metric value is empty")
except Exception as e:
- log.exception("Error collecting metric %s for vdu %s" % (metric_name, vdur['name']))
+ log.exception(
+ "Error collecting metric %s for vdu %s"
+ % (metric_name, vdur["name"])
+ )
log.info("Error in metric collection: %s" % e)
return metrics
@@ -159,12 +178,13 @@
class OpenstackBackend:
- def collect_metric(self, metric_type: MetricType, metric_name: str, resource_id: str):
+ def collect_metric(
+ self, metric_type: MetricType, metric_name: str, resource_id: str
+ ):
pass
class GnocchiBackend(OpenstackBackend):
-
def __init__(self, vim_account: dict):
self.client = self._build_gnocchi_client(vim_account)
self.neutron = self._build_neutron_client(vim_account)
@@ -177,7 +197,9 @@
sess = OpenstackUtils.get_session(vim_account)
return neutron_client.Client(session=sess)
- def collect_metric(self, metric_type: MetricType, metric_name: str, resource_id: str):
+ def collect_metric(
+ self, metric_type: MetricType, metric_name: str, resource_id: str
+ ):
if metric_type == MetricType.INTERFACE_ALL:
return self._collect_interface_all_metric(metric_name, resource_id)
@@ -185,17 +207,19 @@
return self._collect_instance_metric(metric_name, resource_id)
else:
- raise Exception('Unknown metric type %s' % metric_type.value)
+ raise Exception("Unknown metric type %s" % metric_type.value)
def _collect_interface_all_metric(self, openstack_metric_name, resource_id):
total_measure = None
- interfaces = self.client.resource.search(resource_type='instance_network_interface',
- query={'=': {'instance_id': resource_id}})
+ interfaces = self.client.resource.search(
+ resource_type="instance_network_interface",
+ query={"=": {"instance_id": resource_id}},
+ )
for interface in interfaces:
try:
- measures = self.client.metric.get_measures(openstack_metric_name,
- resource_id=interface['id'],
- limit=1)
+ measures = self.client.metric.get_measures(
+ openstack_metric_name, resource_id=interface["id"], limit=1
+ )
if measures:
if not total_measure:
total_measure = 0.0
@@ -203,8 +227,12 @@
except (gnocchiclient.exceptions.NotFound, TypeError) as e:
# Gnocchi in some Openstack versions raise TypeError instead of NotFound
- log.debug("No metric %s found for interface %s: %s", openstack_metric_name,
- interface['id'], e)
+ log.debug(
+ "No metric %s found for interface %s: %s",
+ openstack_metric_name,
+ interface["id"],
+ e,
+ )
return total_measure
def _collect_instance_metric(self, openstack_metric_name, resource_id):
@@ -213,22 +241,36 @@
aggregation = METRIC_AGGREGATORS.get(openstack_metric_name)
try:
- measures = self.client.metric.get_measures(openstack_metric_name,
- aggregation=aggregation,
- start=time.time() - 1200,
- resource_id=resource_id)
+ measures = self.client.metric.get_measures(
+ openstack_metric_name,
+ aggregation=aggregation,
+ start=time.time() - 1200,
+ resource_id=resource_id,
+ )
if measures:
value = measures[-1][2]
- except (gnocchiclient.exceptions.NotFound, gnocchiclient.exceptions.BadRequest, TypeError) as e:
+ except (
+ gnocchiclient.exceptions.NotFound,
+ gnocchiclient.exceptions.BadRequest,
+ TypeError,
+ ) as e:
# CPU metric in previous Openstack versions do not support rate:mean aggregation method
# Gnocchi in some Openstack versions raise TypeError instead of NotFound or BadRequest
if openstack_metric_name == "cpu":
- log.debug("No metric %s found for instance %s: %s", openstack_metric_name, resource_id, e)
- log.info("Retrying to get metric %s for instance %s without aggregation",
- openstack_metric_name, resource_id)
- measures = self.client.metric.get_measures(openstack_metric_name,
- resource_id=resource_id,
- limit=1)
+ log.debug(
+ "No metric %s found for instance %s: %s",
+ openstack_metric_name,
+ resource_id,
+ e,
+ )
+ log.info(
+ "Retrying to get metric %s for instance %s without aggregation",
+ openstack_metric_name,
+ resource_id,
+ )
+ measures = self.client.metric.get_measures(
+ openstack_metric_name, resource_id=resource_id, limit=1
+ )
else:
raise e
# measures[-1] is the last measure
@@ -247,8 +289,12 @@
if openstack_metric_name in METRIC_MULTIPLIERS:
value = value * METRIC_MULTIPLIERS[openstack_metric_name]
except gnocchiclient.exceptions.NotFound as e:
- log.debug("No metric %s found for instance %s: %s", openstack_metric_name, resource_id,
- e)
+ log.debug(
+ "No metric %s found for instance %s: %s",
+ openstack_metric_name,
+ resource_id,
+ e,
+ )
return value
@@ -260,9 +306,16 @@
sess = OpenstackUtils.get_session(vim_account)
return ceilometer_client.Client("2", session=sess)
- def collect_metric(self, metric_type: MetricType, metric_name: str, resource_id: str):
+ def collect_metric(
+ self, metric_type: MetricType, metric_name: str, resource_id: str
+ ):
if metric_type != MetricType.INSTANCE:
- raise NotImplementedError('Ceilometer backend only support instance metrics')
- measures = self.client.samples.list(meter_name=metric_name, limit=1, q=[
- {'field': 'resource_id', 'op': 'eq', 'value': resource_id}])
+ raise NotImplementedError(
+ "Ceilometer backend only support instance metrics"
+ )
+ measures = self.client.samples.list(
+ meter_name=metric_name,
+ limit=1,
+ q=[{"field": "resource_id", "op": "eq", "value": resource_id}],
+ )
return measures[0].counter_volume if measures else None
diff --git a/osm_mon/collector/vnf_collectors/vio.py b/osm_mon/collector/vnf_collectors/vio.py
index e84f5e7..4f7d55a 100644
--- a/osm_mon/collector/vnf_collectors/vio.py
+++ b/osm_mon/collector/vnf_collectors/vio.py
@@ -36,57 +36,63 @@
super().__init__(config, vim_account_id)
self.common_db = CommonDbClient(config)
cfg = self.get_vim_account(vim_account_id)
- self.vrops = vROPS_Helper(vrops_site=cfg['vrops_site'],
- vrops_user=cfg['vrops_user'],
- vrops_password=cfg['vrops_password'])
+ self.vrops = vROPS_Helper(
+ vrops_site=cfg["vrops_site"],
+ vrops_user=cfg["vrops_user"],
+ vrops_password=cfg["vrops_password"],
+ )
def get_vim_account(self, vim_account_id: str):
vim_account_info = self.common_db.get_vim_account(vim_account_id)
- return vim_account_info['config']
+ return vim_account_info["config"]
def collect(self, vnfr: dict):
- vnfd = self.common_db.get_vnfd(vnfr['vnfd-id'])
+ vnfd = self.common_db.get_vnfd(vnfr["vnfd-id"])
vdu_mappings = {}
# Populate extra tags for metrics
- nsr_id = vnfr['nsr-id-ref']
+ nsr_id = vnfr["nsr-id-ref"]
tags = {}
- tags['ns_name'] = self.common_db.get_nsr(nsr_id)['name']
- if vnfr['_admin']['projects_read']:
- tags['project_id'] = vnfr['_admin']['projects_read'][0]
+ tags["ns_name"] = self.common_db.get_nsr(nsr_id)["name"]
+ if vnfr["_admin"]["projects_read"]:
+ tags["project_id"] = vnfr["_admin"]["projects_read"][0]
else:
- tags['project_id'] = ''
+ tags["project_id"] = ""
# Fetch the list of all known resources from vROPS.
resource_list = self.vrops.get_vm_resource_list_from_vrops()
- for vdur in vnfr['vdur']:
+ for vdur in vnfr["vdur"]:
# This avoids errors when vdur records have not been completely filled
- if 'name' not in vdur:
+ if "name" not in vdur:
continue
- vdu = next(
- filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu'])
- )
- if 'monitoring-parameter' not in vdu:
+ vdu = next(filter(lambda vdu: vdu["id"] == vdur["vdu-id-ref"], vnfd["vdu"]))
+ if "monitoring-parameter" not in vdu:
continue
- vim_id = vdur['vim-id']
- vdu_mappings[vim_id] = {'name': vdur['name']}
+ vim_id = vdur["vim-id"]
+ vdu_mappings[vim_id] = {"name": vdur["name"]}
# Map the vROPS instance id to the vim-id so we can look it up.
for resource in resource_list:
- for resourceIdentifier in resource['resourceKey']['resourceIdentifiers']:
- if resourceIdentifier['identifierType']['name'] == 'VMEntityInstanceUUID':
- if resourceIdentifier['value'] != vim_id:
+ for resourceIdentifier in resource["resourceKey"][
+ "resourceIdentifiers"
+ ]:
+ if (
+ resourceIdentifier["identifierType"]["name"]
+ == "VMEntityInstanceUUID"
+ ):
+ if resourceIdentifier["value"] != vim_id:
continue
- vdu_mappings[vim_id]['vrops_id'] = resource['identifier']
+ vdu_mappings[vim_id]["vrops_id"] = resource["identifier"]
if len(vdu_mappings) != 0:
- return self.vrops.get_metrics(vdu_mappings=vdu_mappings,
- monitoring_params=vdu['monitoring-parameter'],
- vnfr=vnfr,
- tags=tags
- )
+ return self.vrops.get_metrics(
+ vdu_mappings=vdu_mappings,
+ monitoring_params=vdu["monitoring-parameter"],
+ vnfr=vnfr,
+ tags=tags,
+ )
else:
return []
diff --git a/osm_mon/collector/vnf_collectors/vmware.py b/osm_mon/collector/vnf_collectors/vmware.py
index 1bc33eb..93592b7 100644
--- a/osm_mon/collector/vnf_collectors/vmware.py
+++ b/osm_mon/collector/vnf_collectors/vmware.py
@@ -36,7 +36,7 @@
log = logging.getLogger(__name__)
-API_VERSION = '27.0'
+API_VERSION = "27.0"
class VMwareCollector(BaseVimCollector):
@@ -44,20 +44,22 @@
super().__init__(config, vim_account_id)
self.common_db = CommonDbClient(config)
vim_account = self.get_vim_account(vim_account_id)
- self.vcloud_site = vim_account['vim_url']
- self.admin_username = vim_account['admin_username']
- self.admin_password = vim_account['admin_password']
- self.vrops = vROPS_Helper(vrops_site=vim_account['vrops_site'],
- vrops_user=vim_account['vrops_user'],
- vrops_password=vim_account['vrops_password'])
+ self.vcloud_site = vim_account["vim_url"]
+ self.admin_username = vim_account["admin_username"]
+ self.admin_password = vim_account["admin_password"]
+ self.vrops = vROPS_Helper(
+ vrops_site=vim_account["vrops_site"],
+ vrops_user=vim_account["vrops_user"],
+ vrops_password=vim_account["vrops_password"],
+ )
def connect_as_admin(self):
- """ Method connect as pvdc admin user to vCloud director.
- There are certain action that can be done only by provider vdc admin user.
- Organization creation / provider network creation etc.
+ """Method connect as pvdc admin user to vCloud director.
+ There are certain action that can be done only by provider vdc admin user.
+ Organization creation / provider network creation etc.
- Returns:
- The return client object that letter can be used to connect to vcloud direct as admin for provider vdc
+ Returns:
+ The return client object that letter can be used to connect to vcloud direct as admin for provider vdc
"""
log.debug("Logging into vCD org as admin.")
@@ -67,41 +69,44 @@
host = self.vcloud_site
admin_user = self.admin_username
admin_passwd = self.admin_password
- org = 'System'
+ org = "System"
client = Client(host, verify_ssl_certs=False)
client.set_highest_supported_version()
- client.set_credentials(BasicLoginCredentials(admin_user, org,
- admin_passwd))
+ client.set_credentials(BasicLoginCredentials(admin_user, org, admin_passwd))
return client
except Exception as e:
- log.error("Can't connect to a vCloud director as: {} with exception {}".format(admin_user, e))
+ log.error(
+ "Can't connect to a vCloud director as: {} with exception {}".format(
+ admin_user, e
+ )
+ )
def get_vim_account(self, vim_account_id: str):
"""
- Method to get VIM account details by its ID
- arg - VIM ID
- return - dict with vim account details
+ Method to get VIM account details by its ID
+ arg - VIM ID
+ return - dict with vim account details
"""
vim_account = {}
vim_account_info = self.common_db.get_vim_account(vim_account_id)
- vim_account['vim_url'] = vim_account_info['vim_url']
+ vim_account["vim_url"] = vim_account_info["vim_url"]
- vim_config = vim_account_info['config']
- vim_account['admin_username'] = vim_config['admin_username']
- vim_account['admin_password'] = vim_config['admin_password']
- vim_account['vrops_site'] = vim_config['vrops_site']
- vim_account['vrops_user'] = vim_config['vrops_user']
- vim_account['vrops_password'] = vim_config['vrops_password']
+ vim_config = vim_account_info["config"]
+ vim_account["admin_username"] = vim_config["admin_username"]
+ vim_account["admin_password"] = vim_config["admin_password"]
+ vim_account["vrops_site"] = vim_config["vrops_site"]
+ vim_account["vrops_user"] = vim_config["vrops_user"]
+ vim_account["vrops_password"] = vim_config["vrops_password"]
return vim_account
def get_vm_moref_id(self, vapp_uuid):
"""
- Method to get the moref_id of given VM
- arg - vapp_uuid
- return - VM mored_id
+ Method to get the moref_id of given VM
+ arg - vapp_uuid
+ return - VM mored_id
"""
vm_moref_id = None
try:
@@ -110,12 +115,24 @@
if vm_details and "vm_vcenter_info" in vm_details:
vm_moref_id = vm_details["vm_vcenter_info"].get("vm_moref_id", None)
- log.debug("Found vm_moref_id: {} for vApp UUID: {}".format(vm_moref_id, vapp_uuid))
+ log.debug(
+ "Found vm_moref_id: {} for vApp UUID: {}".format(
+ vm_moref_id, vapp_uuid
+ )
+ )
else:
- log.error("Failed to find vm_moref_id from vApp UUID: {}".format(vapp_uuid))
+ log.error(
+ "Failed to find vm_moref_id from vApp UUID: {}".format(
+ vapp_uuid
+ )
+ )
except Exception as exp:
- log.warning("Error occurred while getting VM moref ID for VM: {}\n{}".format(exp, traceback.format_exc()))
+ log.warning(
+ "Error occurred while getting VM moref ID for VM: {}\n{}".format(
+ exp, traceback.format_exc()
+ )
+ )
return vm_moref_id
@@ -136,95 +153,117 @@
log.error("Failed to connect to vCD")
return parsed_respond
- url_list = [self.vcloud_site, '/api/vApp/vapp-', vapp_uuid]
- get_vapp_restcall = ''.join(url_list)
+ url_list = [self.vcloud_site, "/api/vApp/vapp-", vapp_uuid]
+ get_vapp_restcall = "".join(url_list)
if vca._session:
- headers = {'Accept': 'application/*+xml;version=' + API_VERSION,
- 'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
- response = requests.get(get_vapp_restcall,
- headers=headers,
- verify=False)
+ headers = {
+ "Accept": "application/*+xml;version=" + API_VERSION,
+ "x-vcloud-authorization": vca._session.headers[
+ "x-vcloud-authorization"
+ ],
+ }
+ response = requests.get(get_vapp_restcall, headers=headers, verify=False)
if response.status_code != 200:
- log.error("REST API call {} failed. Return status code {}".format(get_vapp_restcall,
- response.content))
+ log.error(
+ "REST API call {} failed. Return status code {}".format(
+ get_vapp_restcall, response.content
+ )
+ )
return parsed_respond
try:
xmlroot_respond = XmlElementTree.fromstring(response.content)
- namespaces = {'vm': 'http://www.vmware.com/vcloud/v1.5',
- "vmext": "http://www.vmware.com/vcloud/extension/v1.5",
- "xmlns": "http://www.vmware.com/vcloud/v1.5"}
+ namespaces = {
+ "vm": "http://www.vmware.com/vcloud/v1.5",
+ "vmext": "http://www.vmware.com/vcloud/extension/v1.5",
+ "xmlns": "http://www.vmware.com/vcloud/v1.5",
+ }
# parse children section for other attrib
- children_section = xmlroot_respond.find('vm:Children/', namespaces)
+ children_section = xmlroot_respond.find("vm:Children/", namespaces)
if children_section is not None:
- vCloud_extension_section = children_section.find('xmlns:VCloudExtension', namespaces)
+ vCloud_extension_section = children_section.find(
+ "xmlns:VCloudExtension", namespaces
+ )
if vCloud_extension_section is not None:
vm_vcenter_info = {}
- vim_info = vCloud_extension_section.find('vmext:VmVimInfo', namespaces)
- vmext = vim_info.find('vmext:VmVimObjectRef', namespaces)
+ vim_info = vCloud_extension_section.find(
+ "vmext:VmVimInfo", namespaces
+ )
+ vmext = vim_info.find("vmext:VmVimObjectRef", namespaces)
if vmext is not None:
- vm_vcenter_info["vm_moref_id"] = vmext.find('vmext:MoRef', namespaces).text
+ vm_vcenter_info["vm_moref_id"] = vmext.find(
+ "vmext:MoRef", namespaces
+ ).text
parsed_respond["vm_vcenter_info"] = vm_vcenter_info
except Exception as exp:
- log.warning("Error occurred for getting vApp details: {}\n{}".format(exp,
- traceback.format_exc()))
+ log.warning(
+ "Error occurred for getting vApp details: {}\n{}".format(
+ exp, traceback.format_exc()
+ )
+ )
return parsed_respond
def collect(self, vnfr: dict):
- vnfd = self.common_db.get_vnfd(vnfr['vnfd-id'])
+ vnfd = self.common_db.get_vnfd(vnfr["vnfd-id"])
vdu_mappings = {}
# Populate extra tags for metrics
- nsr_id = vnfr['nsr-id-ref']
+ nsr_id = vnfr["nsr-id-ref"]
tags = {}
- tags['ns_name'] = self.common_db.get_nsr(nsr_id)['name']
- if vnfr['_admin']['projects_read']:
- tags['project_id'] = vnfr['_admin']['projects_read'][0]
+ tags["ns_name"] = self.common_db.get_nsr(nsr_id)["name"]
+ if vnfr["_admin"]["projects_read"]:
+ tags["project_id"] = vnfr["_admin"]["projects_read"][0]
else:
- tags['project_id'] = ''
+ tags["project_id"] = ""
# Fetch the list of all known resources from vROPS.
resource_list = self.vrops.get_vm_resource_list_from_vrops()
- for vdur in vnfr['vdur']:
+ for vdur in vnfr["vdur"]:
# This avoids errors when vdur records have not been completely filled
- if 'name' not in vdur:
+ if "name" not in vdur:
continue
- vdu = next(
- filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu'])
- )
+ vdu = next(filter(lambda vdu: vdu["id"] == vdur["vdu-id-ref"], vnfd["vdu"]))
- if 'monitoring-parameter' not in vdu:
+ if "monitoring-parameter" not in vdu:
continue
- resource_uuid = vdur['vim-id']
+ resource_uuid = vdur["vim-id"]
# Find vm_moref_id from vApp uuid in vCD
vim_id = self.get_vm_moref_id(resource_uuid)
if vim_id is None:
- log.debug("Failed to find vROPS ID for vApp in vCD: {}".format(resource_uuid))
+ log.debug(
+ "Failed to find vROPS ID for vApp in vCD: {}".format(resource_uuid)
+ )
continue
- vdu_mappings[vim_id] = {'name': vdur['name']}
+ vdu_mappings[vim_id] = {"name": vdur["name"]}
# Map the vROPS instance id to the vim-id so we can look it up.
for resource in resource_list:
- for resourceIdentifier in resource['resourceKey']['resourceIdentifiers']:
- if resourceIdentifier['identifierType']['name'] == 'VMEntityObjectID':
- if resourceIdentifier['value'] != vim_id:
+ for resourceIdentifier in resource["resourceKey"][
+ "resourceIdentifiers"
+ ]:
+ if (
+ resourceIdentifier["identifierType"]["name"]
+ == "VMEntityObjectID"
+ ):
+ if resourceIdentifier["value"] != vim_id:
continue
- vdu_mappings[vim_id]['vrops_id'] = resource['identifier']
+ vdu_mappings[vim_id]["vrops_id"] = resource["identifier"]
if len(vdu_mappings) != 0:
- return self.vrops.get_metrics(vdu_mappings=vdu_mappings,
- monitoring_params=vdu['monitoring-parameter'],
- vnfr=vnfr,
- tags=tags
- )
+ return self.vrops.get_metrics(
+ vdu_mappings=vdu_mappings,
+ monitoring_params=vdu["monitoring-parameter"],
+ vnfr=vnfr,
+ tags=tags,
+ )
else:
return []
diff --git a/osm_mon/collector/vnf_collectors/vrops/vrops_helper.py b/osm_mon/collector/vnf_collectors/vrops/vrops_helper.py
index 775aa10..2196326 100644
--- a/osm_mon/collector/vnf_collectors/vrops/vrops_helper.py
+++ b/osm_mon/collector/vnf_collectors/vrops/vrops_helper.py
@@ -39,73 +39,75 @@
"disk_read_bytes": 1024,
"disk_write_bytes": 1024,
"packets_received": 1024,
- "packets_sent": 1024
+ "packets_sent": 1024,
}
-class vROPS_Helper():
-
- def __init__(self,
- vrops_site='https://vrops',
- vrops_user='',
- vrops_password=''):
+class vROPS_Helper:
+ def __init__(self, vrops_site="https://vrops", vrops_user="", vrops_password=""):
self.vrops_site = vrops_site
self.vrops_user = vrops_user
self.vrops_password = vrops_password
def get_vrops_token(self):
- """Fetches token from vrops
- """
- auth_url = '/suite-api/api/auth/token/acquire'
- headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
+ """Fetches token from vrops"""
+ auth_url = "/suite-api/api/auth/token/acquire"
+ headers = {"Content-Type": "application/json", "Accept": "application/json"}
req_body = {"username": self.vrops_user, "password": self.vrops_password}
- resp = requests.post(self.vrops_site + auth_url,
- json=req_body, verify=False,
- headers=headers)
+ resp = requests.post(
+ self.vrops_site + auth_url, json=req_body, verify=False, headers=headers
+ )
if resp.status_code != 200:
- log.error("Failed to get token from vROPS: {} {}".format(resp.status_code,
- resp.content))
+ log.error(
+ "Failed to get token from vROPS: {} {}".format(
+ resp.status_code, resp.content
+ )
+ )
return None
- resp_data = json.loads(resp.content.decode('utf-8'))
- return resp_data['token']
+ resp_data = json.loads(resp.content.decode("utf-8"))
+ return resp_data["token"]
def get_vm_resource_list_from_vrops(self):
- """ Find all known resource IDs in vROPs
- """
+ """Find all known resource IDs in vROPs"""
auth_token = self.get_vrops_token()
- api_url = '/suite-api/api/resources?resourceKind=VirtualMachine'
- headers = {'Accept': 'application/json', 'Authorization': 'vRealizeOpsToken {}'.format(auth_token)}
+ api_url = "/suite-api/api/resources?resourceKind=VirtualMachine"
+ headers = {
+ "Accept": "application/json",
+ "Authorization": "vRealizeOpsToken {}".format(auth_token),
+ }
resource_list = []
- resp = requests.get(self.vrops_site + api_url,
- verify=False, headers=headers)
+ resp = requests.get(self.vrops_site + api_url, verify=False, headers=headers)
if resp.status_code != 200:
- log.error("Failed to get resource list from vROPS: {} {}".format(resp.status_code,
- resp.content))
+ log.error(
+ "Failed to get resource list from vROPS: {} {}".format(
+ resp.status_code, resp.content
+ )
+ )
return resource_list
try:
- resp_data = json.loads(resp.content.decode('utf-8'))
- if resp_data.get('resourceList') is not None:
- resource_list = resp_data.get('resourceList')
+ resp_data = json.loads(resp.content.decode("utf-8"))
+ if resp_data.get("resourceList") is not None:
+ resource_list = resp_data.get("resourceList")
except Exception as exp:
- log.error("get_vm_resource_id: Error in parsing {}\n{}".format(exp, traceback.format_exc()))
+ log.error(
+ "get_vm_resource_id: Error in parsing {}\n{}".format(
+ exp, traceback.format_exc()
+ )
+ )
return resource_list
- def get_metrics(self,
- vdu_mappings={},
- monitoring_params={},
- vnfr=None,
- tags={}):
+ def get_metrics(self, vdu_mappings={}, monitoring_params={}, vnfr=None, tags={}):
monitoring_keys = {}
# Collect the names of all the metrics we need to query
for metric_entry in monitoring_params:
- metric_name = metric_entry['performance-metric']
+ metric_name = metric_entry["performance-metric"]
if metric_name not in METRIC_MAPPINGS:
log.debug("Metric {} not supported, ignoring".format(metric_name))
continue
@@ -122,66 +124,81 @@
sanitized_vdu_mappings = copy.deepcopy(vdu_mappings)
for key in vdu_mappings.keys():
vdu = vdu_mappings[key]
- if 'vrops_id' not in vdu:
+ if "vrops_id" not in vdu:
log.info("Could not find vROPS id for vdu {}".format(vdu))
del sanitized_vdu_mappings[key]
continue
- resource_ids += "&resourceId={}".format(vdu['vrops_id'])
+ resource_ids += "&resourceId={}".format(vdu["vrops_id"])
vdu_mappings = sanitized_vdu_mappings
try:
# Now we can make a single call to vROPS to collect all relevant metrics for resources we need to monitor
- api_url = "/suite-api/api/resources/stats?IntervalType=MINUTES&IntervalCount=1"\
+ api_url = (
+ "/suite-api/api/resources/stats?IntervalType=MINUTES&IntervalCount=1"
"&rollUpType=MAX¤tOnly=true{}{}".format(stats_key, resource_ids)
+ )
auth_token = self.get_vrops_token()
- headers = {'Accept': 'application/json', 'Authorization': 'vRealizeOpsToken {}'.format(auth_token)}
+ headers = {
+ "Accept": "application/json",
+ "Authorization": "vRealizeOpsToken {}".format(auth_token),
+ }
- resp = requests.get(self.vrops_site + api_url,
- verify=False, headers=headers)
+ resp = requests.get(
+ self.vrops_site + api_url, verify=False, headers=headers
+ )
if resp.status_code != 200:
- log.error("Failed to get Metrics data from vROPS for {} {}".format(resp.status_code,
- resp.content))
+ log.error(
+ "Failed to get Metrics data from vROPS for {} {}".format(
+ resp.status_code, resp.content
+ )
+ )
return metrics
- m_data = json.loads(resp.content.decode('utf-8'))
- if 'values' not in m_data:
+ m_data = json.loads(resp.content.decode("utf-8"))
+ if "values" not in m_data:
return metrics
- statistics = m_data['values']
+ statistics = m_data["values"]
for vdu_stat in statistics:
- vrops_id = vdu_stat['resourceId']
+ vrops_id = vdu_stat["resourceId"]
vdu_name = None
for vdu in vdu_mappings.values():
- if vdu['vrops_id'] == vrops_id:
- vdu_name = vdu['name']
+ if vdu["vrops_id"] == vrops_id:
+ vdu_name = vdu["name"]
if vdu_name is None:
continue
- for item in vdu_stat['stat-list']['stat']:
- reported_metric = item['statKey']['key']
+ for item in vdu_stat["stat-list"]["stat"]:
+ reported_metric = item["statKey"]["key"]
if reported_metric not in METRIC_MAPPINGS.values():
continue
# Convert the vROPS metric name back to OSM key
- metric_name = list(METRIC_MAPPINGS.keys())[list(METRIC_MAPPINGS.values()).
- index(reported_metric)]
+ metric_name = list(METRIC_MAPPINGS.keys())[
+ list(METRIC_MAPPINGS.values()).index(reported_metric)
+ ]
if metric_name in monitoring_keys.keys():
- metric_value = item['data'][-1]
+ metric_value = item["data"][-1]
if metric_name in METRIC_MULTIPLIERS:
metric_value *= METRIC_MULTIPLIERS[metric_name]
- metric = VnfMetric(vnfr['nsr-id-ref'],
- vnfr['member-vnf-index-ref'],
- vdu_name,
- metric_name,
- metric_value,
- tags
- )
+ metric = VnfMetric(
+ vnfr["nsr-id-ref"],
+ vnfr["member-vnf-index-ref"],
+ vdu_name,
+ metric_name,
+ metric_value,
+ tags,
+ )
metrics.append(metric)
except Exception as exp:
- log.error("Exception while parsing metrics data from vROPS {}\n{}".format(exp, traceback.format_exc()))
+ log.error(
+ "Exception while parsing metrics data from vROPS {}\n{}".format(
+ exp, traceback.format_exc()
+ )
+ )
return metrics
diff --git a/osm_mon/collector/vnf_metric.py b/osm_mon/collector/vnf_metric.py
index 49e7842..961a4ef 100644
--- a/osm_mon/collector/vnf_metric.py
+++ b/osm_mon/collector/vnf_metric.py
@@ -26,13 +26,15 @@
class VnfMetric(Metric):
- def __init__(self, nsr_id, vnf_member_index, vdur_name, name, value, extra_tags: dict = None):
+ def __init__(
+ self, nsr_id, vnf_member_index, vdur_name, name, value, extra_tags: dict = None
+ ):
tags = {
- 'ns_id': nsr_id,
- 'vnf_member_index': vnf_member_index,
- 'vdu_name': vdur_name
+ "ns_id": nsr_id,
+ "vnf_member_index": vnf_member_index,
+ "vdu_name": vdur_name,
}
if extra_tags:
tags.update(extra_tags)
- log.debug('Tags: %s', tags)
+ log.debug("Tags: %s", tags)
super().__init__(tags, name, value)