Revert "Feature 11061: Clean MON to have only the dashboarder"
This reverts commit 8c141fe62eb1d1f097c4d1d92f19a19a5de22d20.
Change-Id: I9c83c3c832312915439c7aa4b5ad5e081163bfa4
Signed-off-by: garciadeblas <gerardo.garciadeblas@telefonica.com>
diff --git a/osm_mon/collector/__init__.py b/osm_mon/collector/__init__.py
new file mode 100644
index 0000000..d81308a
--- /dev/null
+++ b/osm_mon/collector/__init__.py
@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
diff --git a/osm_mon/collector/backends/__init__.py b/osm_mon/collector/backends/__init__.py
new file mode 100644
index 0000000..971f4e9
--- /dev/null
+++ b/osm_mon/collector/backends/__init__.py
@@ -0,0 +1,21 @@
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
diff --git a/osm_mon/collector/backends/base.py b/osm_mon/collector/backends/base.py
new file mode 100644
index 0000000..8cba5e1
--- /dev/null
+++ b/osm_mon/collector/backends/base.py
@@ -0,0 +1,26 @@
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+
+
+class BaseBackend:
+ def handle(self, metrics: list):
+ pass
diff --git a/osm_mon/collector/backends/prometheus.py b/osm_mon/collector/backends/prometheus.py
new file mode 100644
index 0000000..a9bb938
--- /dev/null
+++ b/osm_mon/collector/backends/prometheus.py
@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+import logging
+from typing import List
+
+from prometheus_client import start_http_server
+from prometheus_client.core import REGISTRY, GaugeMetricFamily
+
+from osm_mon.collector.backends.base import BaseBackend
+from osm_mon.collector.metric import Metric
+
+log = logging.getLogger(__name__)
+
+OSM_METRIC_PREFIX = "osm_"
+
+
+class PrometheusBackend(BaseBackend):
+ def __init__(self):
+ self.custom_collector = CustomCollector()
+ self._start_exporter(8000)
+
+ def handle(self, metrics: List[Metric]):
+ log.debug("handle")
+ log.debug("metrics: %s", metrics)
+ prometheus_metrics = {}
+ for metric in metrics:
+ if metric.name not in prometheus_metrics:
+ prometheus_metrics[metric.name] = GaugeMetricFamily(
+ OSM_METRIC_PREFIX + metric.name,
+ "OSM metric",
+ labels=list(metric.tags.keys()),
+ )
+ prometheus_metrics[metric.name].add_metric(
+ list(metric.tags.values()), metric.value
+ )
+ self.custom_collector.metrics = prometheus_metrics.values()
+
+ def _start_exporter(self, port):
+ log.debug("_start_exporter")
+ log.debug("port: %s", port)
+ REGISTRY.register(self.custom_collector)
+ log.info("Starting MON Prometheus exporter at port %s", port)
+ start_http_server(port)
+
+
+class CustomCollector(object):
+ def __init__(self):
+ self.metrics = []
+
+ def describe(self):
+ log.debug("describe")
+ return []
+
+ def collect(self):
+ log.debug("collect")
+ return self.metrics
diff --git a/osm_mon/collector/collector.py b/osm_mon/collector/collector.py
new file mode 100644
index 0000000..a69e651
--- /dev/null
+++ b/osm_mon/collector/collector.py
@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+import logging
+import time
+
+from osm_mon.collector.backends.prometheus import PrometheusBackend
+from osm_mon.collector.service import CollectorService
+from osm_mon.core.config import Config
+
+log = logging.getLogger(__name__)
+
+METRIC_BACKENDS = [PrometheusBackend]
+
+
+class Collector:
+ def __init__(self, config: Config):
+ self.conf = config
+ self.service = CollectorService(config)
+ self.backends = []
+ self._init_backends()
+
+ def collect_forever(self):
+ log.debug("collect_forever")
+ while True:
+ try:
+ self.collect_metrics()
+ time.sleep(int(self.conf.get("collector", "interval")))
+ except Exception:
+ log.exception("Error collecting metrics")
+
+ def collect_metrics(self):
+ metrics = self.service.collect_metrics()
+ for backend in self.backends:
+ backend.handle(metrics)
+
+ def _init_backends(self):
+ for backend in METRIC_BACKENDS:
+ self.backends.append(backend())
diff --git a/osm_mon/collector/infra_collectors/__init__.py b/osm_mon/collector/infra_collectors/__init__.py
new file mode 100644
index 0000000..971f4e9
--- /dev/null
+++ b/osm_mon/collector/infra_collectors/__init__.py
@@ -0,0 +1,21 @@
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
diff --git a/osm_mon/collector/infra_collectors/base.py b/osm_mon/collector/infra_collectors/base.py
new file mode 100644
index 0000000..2f97ebc
--- /dev/null
+++ b/osm_mon/collector/infra_collectors/base.py
@@ -0,0 +1,29 @@
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+from typing import List
+
+from osm_mon.collector.metric import Metric
+
+
+class BaseInfraCollector:
+ def collect(self) -> List[Metric]:
+ pass
diff --git a/osm_mon/collector/infra_collectors/base_osinfra.py b/osm_mon/collector/infra_collectors/base_osinfra.py
new file mode 100644
index 0000000..8ab34e4
--- /dev/null
+++ b/osm_mon/collector/infra_collectors/base_osinfra.py
@@ -0,0 +1,294 @@
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+import logging
+from typing import List
+
+from keystoneclient.v3 import client as keystone_client
+from novaclient import client as nova_client
+from cinderclient import client as cinder_client
+from neutronclient.neutron import client as neutron_client
+
+from osm_mon.collector.infra_collectors.base_vim import BaseVimInfraCollector
+from osm_mon.collector.metric import Metric
+from osm_mon.collector.utils.openstack import OpenstackUtils
+from osm_mon.core.common_db import CommonDbClient
+from osm_mon.core.config import Config
+
+log = logging.getLogger(__name__)
+
+
+class BaseOpenStackInfraCollector(BaseVimInfraCollector):
+ def __init__(self, config: Config, vim_account_id: str):
+ super().__init__(config, vim_account_id)
+ self.conf = config
+ self.common_db = CommonDbClient(config)
+ self.vim_account = self.common_db.get_vim_account(vim_account_id)
+ # self.keystone = self._build_keystone_client(self.vim_account)
+ self.vim_session = None
+ self.nova = self._build_nova_client(self.vim_account)
+ self.cinder = self._build_cinder_client(self.vim_account)
+ self.neutron, self.tenant_id = self._build_neutron_client(self.vim_account)
+
+ def collect(self) -> List[Metric]:
+ metrics = []
+ vim_status = self.is_vim_ok()
+ if vim_status:
+ # Updating the resources in mongoDB
+ self.update_resources()
+ if self.vim_account["_admin"]["projects_read"]:
+ vim_project_id = self.vim_account["_admin"]["projects_read"][0]
+ else:
+ vim_project_id = ""
+ vim_tags = {
+ "vim_account_id": self.vim_account["_id"],
+ "project_id": vim_project_id,
+ }
+ vim_status_metric = Metric(vim_tags, "vim_status", vim_status)
+ metrics.append(vim_status_metric)
+ vnfrs = self.common_db.get_vnfrs(vim_account_id=self.vim_account["_id"])
+ if self.conf.get("collector", "vm_infra_metrics"):
+ vm_infra_metrics_enabled = str(
+ self.conf.get("collector", "vm_infra_metrics")
+ ).lower() in ("yes", "true", "1")
+ else:
+ vm_infra_metrics_enabled = True
+ if vm_infra_metrics_enabled:
+ for vnfr in vnfrs:
+ nsr_id = vnfr["nsr-id-ref"]
+ ns_name = self.common_db.get_nsr(nsr_id)["name"]
+ vnf_member_index = vnfr["member-vnf-index-ref"]
+ if vnfr["_admin"]["projects_read"]:
+ vnfr_project_id = vnfr["_admin"]["projects_read"][0]
+ else:
+ vnfr_project_id = ""
+ for vdur in vnfr["vdur"]:
+ if "vim-id" not in vdur:
+ log.debug("Field vim-id is not present in vdur")
+ continue
+ resource_uuid = vdur["vim-id"]
+ tags = {
+ "vim_account_id": self.vim_account["_id"],
+ "resource_uuid": resource_uuid,
+ "ns_id": nsr_id,
+ "ns_name": ns_name,
+ "vnf_member_index": vnf_member_index,
+ "vdu_name": vdur.get("name", ""),
+ "project_id": vnfr_project_id,
+ }
+ try:
+ vm = self.nova.servers.get(resource_uuid)
+ vm_status = 0 if (vm.status == "ERROR") else 1
+ vm_status_metric = Metric(tags, "vm_status", vm_status)
+ except Exception as e:
+ log.warning("VM status is not OK: %s" % e)
+ vm_status_metric = Metric(tags, "vm_status", 0)
+ metrics.append(vm_status_metric)
+
+ return metrics
+
+ def is_vim_ok(self) -> bool:
+ try:
+ self.nova.servers.list()
+ return True
+ except Exception as e:
+ log.warning("VIM status is not OK: %s" % e)
+ return False
+
+ def update_resources(self):
+ if "resources" in self.vim_account:
+ vimacc_resources = self.vim_account["resources"]
+ # Compute resources
+ try:
+ com_lim = self.nova.limits.get()._info["absolute"]
+ if ("compute" in vimacc_resources) and (
+ (
+ vimacc_resources["compute"]["ram"]["total"]
+ != com_lim["maxTotalRAMSize"]
+ )
+ or (
+ vimacc_resources["compute"]["vcpus"]["total"]
+ != com_lim["maxTotalCores"]
+ )
+ or (
+ vimacc_resources["compute"]["ram"]["used"]
+ != com_lim["totalRAMUsed"]
+ )
+ or (
+ vimacc_resources["compute"]["vcpus"]["used"]
+ != com_lim["totalCoresUsed"]
+ )
+ or (
+ vimacc_resources["compute"]["instances"]["total"]
+ != com_lim["maxTotalInstances"]
+ )
+ or (
+ vimacc_resources["compute"]["instances"]["used"]
+ != com_lim["totalInstancesUsed"]
+ )
+ ):
+ update_dict = {
+ "resources.compute": {
+ "ram": {
+ "total": com_lim["maxTotalRAMSize"],
+ "used": com_lim["totalRAMUsed"],
+ },
+ "vcpus": {
+ "total": com_lim["maxTotalCores"],
+ "used": com_lim["totalCoresUsed"],
+ },
+ "instances": {
+ "total": com_lim["maxTotalInstances"],
+ "used": com_lim["totalInstancesUsed"],
+ },
+ }
+ }
+ suc_value = self.common_db.set_vim_account(
+ str(self.vim_account["_id"]), update_dict
+ )
+ log.info("Compute resources update in mongoDB = %s" % suc_value)
+ except Exception as e:
+ log.warning("Error in updating compute resources: %s" % e)
+
+ # Volume resources
+ try:
+ vol_lim = self.cinder.limits.get()._info["absolute"]
+ if ("storage" in vimacc_resources) and (
+ (
+ vimacc_resources["storage"]["volumes"]["total"]
+ != vol_lim["maxTotalVolumes"]
+ )
+ or (
+ vimacc_resources["storage"]["snapshots"]["total"]
+ != vol_lim["maxTotalSnapshots"]
+ )
+ or (
+ vimacc_resources["storage"]["volumes"]["used"]
+ != vol_lim["totalVolumesUsed"]
+ )
+ or (
+ vimacc_resources["storage"]["snapshots"]["used"]
+ != vol_lim["totalSnapshotsUsed"]
+ )
+ or (
+ vimacc_resources["storage"]["storage"]["total"]
+ != vol_lim["maxTotalVolumeGigabytes"]
+ )
+ or (
+ vimacc_resources["storage"]["storage"]["used"]
+ != vol_lim["totalGigabytesUsed"]
+ )
+ ):
+ update_dict = {
+ "resources.storage": {
+ "volumes": {
+ "total": vol_lim["maxTotalVolumes"],
+ "used": vol_lim["totalVolumesUsed"],
+ },
+ "snapshots": {
+ "total": vol_lim["maxTotalSnapshots"],
+ "used": vol_lim["totalSnapshotsUsed"],
+ },
+ "storage": {
+ "total": vol_lim["maxTotalVolumeGigabytes"],
+ "used": vol_lim["totalGigabytesUsed"],
+ },
+ }
+ }
+ suc_value = self.common_db.set_vim_account(
+ str(self.vim_account["_id"]), update_dict
+ )
+ log.info("Volume resources update in mongoDB = %s" % suc_value)
+ except Exception as e:
+ log.warning("Error in updating volume resources: %s" % e)
+
+ # Network resources
+ try:
+ net_lim = self.neutron.show_quota_details(self.tenant_id)["quota"]
+ if ("network" in vimacc_resources) and (
+ (
+ vimacc_resources["network"]["networks"]["total"]
+ != net_lim["network"]["limit"]
+ )
+ or (
+ vimacc_resources["network"]["networks"]["used"]
+ != net_lim["network"]["used"]
+ )
+ or (
+ vimacc_resources["network"]["subnets"]["total"]
+ != net_lim["subnet"]["limit"]
+ )
+ or (
+ vimacc_resources["network"]["subnets"]["used"]
+ != net_lim["subnet"]["used"]
+ )
+ or (
+ vimacc_resources["network"]["floating_ips"]["total"]
+ != net_lim["floatingip"]["limit"]
+ )
+ or (
+ vimacc_resources["network"]["floating_ips"]["used"]
+ != net_lim["floatingip"]["used"]
+ )
+ ):
+ update_dict = {
+ "resources.network": {
+ "networks": {
+ "total": net_lim["network"]["limit"],
+ "used": net_lim["network"]["used"],
+ },
+ "subnets": {
+ "total": net_lim["subnet"]["limit"],
+ "used": net_lim["subnet"]["used"],
+ },
+ "floating_ips": {
+ "total": net_lim["floatingip"]["limit"],
+ "used": net_lim["floatingip"]["used"],
+ },
+ }
+ }
+ suc_value = self.common_db.set_vim_account(
+ str(self.vim_account["_id"]), update_dict
+ )
+ log.info("Network resources update in mongoDB = %s" % suc_value)
+ except Exception as e:
+ log.warning("Error in updating network resources: %s" % e)
+
+ def _build_keystone_client(self, vim_account: dict) -> keystone_client.Client:
+ sess = OpenstackUtils.get_session(vim_account)
+ return keystone_client.Client(session=sess, timeout=10)
+
+ def _build_nova_client(self, vim_account: dict) -> nova_client.Client:
+ sess = OpenstackUtils.get_session(vim_account)
+ self.vim_session = sess
+ return nova_client.Client("2", session=sess, timeout=10)
+
+ def _build_cinder_client(self, vim_account: dict) -> cinder_client.Client:
+ # sess = OpenstackUtils.get_session(vim_account)
+ return cinder_client.Client("3", session=self.vim_session, timeout=10)
+
+ def _build_neutron_client(self, vim_account: dict) -> tuple:
+ # sess = OpenstackUtils.get_session(vim_account)
+ tenant_id = self.vim_session.get_project_id()
+ return (
+ neutron_client.Client("2", session=self.vim_session, timeout=10),
+ tenant_id,
+ )
diff --git a/osm_mon/collector/infra_collectors/base_sdnc.py b/osm_mon/collector/infra_collectors/base_sdnc.py
new file mode 100644
index 0000000..e46f984
--- /dev/null
+++ b/osm_mon/collector/infra_collectors/base_sdnc.py
@@ -0,0 +1,29 @@
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+from osm_mon.collector.infra_collectors.base import BaseInfraCollector
+
+from osm_mon.core.config import Config
+
+
+class BaseSdncInfraCollector(BaseInfraCollector):
+ def __init__(self, config: Config, sdn_id: str):
+ pass
diff --git a/osm_mon/collector/infra_collectors/base_vim.py b/osm_mon/collector/infra_collectors/base_vim.py
new file mode 100644
index 0000000..698dde4
--- /dev/null
+++ b/osm_mon/collector/infra_collectors/base_vim.py
@@ -0,0 +1,29 @@
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+from osm_mon.collector.infra_collectors.base import BaseInfraCollector
+
+from osm_mon.core.config import Config
+
+
+class BaseVimInfraCollector(BaseInfraCollector):
+ def __init__(self, config: Config, vim_account_id: str):
+ pass
diff --git a/osm_mon/collector/infra_collectors/onos.py b/osm_mon/collector/infra_collectors/onos.py
new file mode 100644
index 0000000..1798f8b
--- /dev/null
+++ b/osm_mon/collector/infra_collectors/onos.py
@@ -0,0 +1,79 @@
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+import logging
+from typing import List
+
+import requests
+from requests.auth import HTTPBasicAuth
+
+from osm_mon.collector.infra_collectors.base_sdnc import BaseSdncInfraCollector
+from osm_mon.collector.metric import Metric
+from osm_mon.core.common_db import CommonDbClient
+from osm_mon.core.config import Config
+
+log = logging.getLogger(__name__)
+
+
+class OnosInfraCollector(BaseSdncInfraCollector):
+ def __init__(self, config: Config, sdnc_id: str):
+ super().__init__(config, sdnc_id)
+ self.common_db = CommonDbClient(config)
+ self.sdnc = self.common_db.get_sdnc(sdnc_id)
+
+ def _obtain_url(self, sdnc_dict):
+ url = sdnc_dict.get("url")
+ if url:
+ return url
+ else:
+ if not sdnc_dict.get("ip") or not sdnc_dict.get("port"):
+ raise Exception("You must provide a URL to contact the SDN Controller")
+ else:
+ return "http://{}:{}/onos/v1/devices".format(
+ sdnc_dict["ip"], sdnc_dict["port"]
+ )
+
+ def collect(self) -> List[Metric]:
+ metrics = []
+ sdnc_status = self.is_sdnc_ok()
+ if self.sdnc["_admin"]["projects_read"]:
+ sdnc_project_id = self.sdnc["_admin"]["projects_read"][0]
+ else:
+ sdnc_project_id = ""
+ sdnc_tags = {"sdnc_id": self.sdnc["_id"], "project_id": sdnc_project_id}
+ sdnc_status_metric = Metric(sdnc_tags, "sdnc_status", sdnc_status)
+ metrics.append(sdnc_status_metric)
+
+ return metrics
+
+ def is_sdnc_ok(self) -> bool:
+ try:
+ url = self._obtain_url(self.sdnc)
+ user = self.sdnc["user"]
+ password = self.common_db.decrypt_sdnc_password(
+ self.sdnc["password"], self.sdnc["schema_version"], self.sdnc["_id"]
+ )
+
+ requests.get(url, auth=HTTPBasicAuth(user, password))
+ return True
+ except Exception:
+ log.exception("SDNC status is not OK!")
+ return False
diff --git a/osm_mon/collector/infra_collectors/openstack.py b/osm_mon/collector/infra_collectors/openstack.py
new file mode 100644
index 0000000..577bf06
--- /dev/null
+++ b/osm_mon/collector/infra_collectors/openstack.py
@@ -0,0 +1,30 @@
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+
+from osm_mon.collector.infra_collectors.base_osinfra import BaseOpenStackInfraCollector
+
+from osm_mon.core.config import Config
+
+
+class OpenstackInfraCollector(BaseOpenStackInfraCollector):
+ def __init__(self, config: Config, vim_account_id: str):
+ super(OpenstackInfraCollector, self).__init__(config, vim_account_id)
diff --git a/osm_mon/collector/infra_collectors/vio.py b/osm_mon/collector/infra_collectors/vio.py
new file mode 100644
index 0000000..d8d8fcc
--- /dev/null
+++ b/osm_mon/collector/infra_collectors/vio.py
@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+
+##
+# Copyright 2016-2017 VMware Inc.
+# This file is part of ETSI OSM
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: osslegalrouting@vmware.com
+##
+
+from osm_mon.collector.infra_collectors.base_osinfra import BaseOpenStackInfraCollector
+
+from osm_mon.core.config import Config
+
+
+class VIOInfraCollector(BaseOpenStackInfraCollector):
+ def __init__(self, config: Config, vim_account_id: str):
+ super(VIOInfraCollector, self).__init__(config, vim_account_id)
diff --git a/osm_mon/collector/infra_collectors/vmware.py b/osm_mon/collector/infra_collectors/vmware.py
new file mode 100644
index 0000000..8f39464
--- /dev/null
+++ b/osm_mon/collector/infra_collectors/vmware.py
@@ -0,0 +1,241 @@
+# -*- coding: utf-8 -*-
+
+##
+# Copyright 2016-2019 VMware Inc.
+# This file is part of ETSI OSM
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: osslegalrouting@vmware.com
+##
+
+# pylint: disable=E1101
+
+import logging
+from typing import List
+from lxml import etree as XmlElementTree
+
+import requests
+from pyvcloud.vcd.client import BasicLoginCredentials
+from pyvcloud.vcd.client import Client
+
+from osm_mon.collector.infra_collectors.base_vim import BaseVimInfraCollector
+from osm_mon.collector.metric import Metric
+from osm_mon.core.common_db import CommonDbClient
+from osm_mon.core.config import Config
+
+log = logging.getLogger(__name__)
+API_VERSION = "27.0"
+
+
+class VMwareInfraCollector(BaseVimInfraCollector):
+ def __init__(self, config: Config, vim_account_id: str):
+ super().__init__(config, vim_account_id)
+ self.vim_account_id = vim_account_id
+ self.common_db = CommonDbClient(config)
+ vim_account = self.get_vim_account(vim_account_id)
+ self.vcloud_site = vim_account["vim_url"]
+ self.admin_username = vim_account["admin_username"]
+ self.admin_password = vim_account["admin_password"]
+ self.vim_uuid = vim_account["vim_uuid"]
+ self.org_name = vim_account["orgname"]
+ self.vim_project_id = vim_account["project_id"]
+ self.verify_ssl = vim_account.get("insecure", False)
+
+ def connect_vim_as_admin(self):
+ """Method connect as pvdc admin user to vCloud director.
+ There are certain action that can be done only by provider vdc admin user.
+ Organization creation / provider network creation etc.
+
+ Returns:
+ The return client object that letter can be used to connect to vcloud direct as admin for provider vdc
+ """
+
+ log.info("Logging into vCD org as admin.")
+
+ admin_user = None
+ try:
+ host = self.vcloud_site
+ admin_user = self.admin_username
+ admin_passwd = self.admin_password
+ org = "System"
+ client = Client(host, verify_ssl_certs=self.verify_ssl)
+ client.set_highest_supported_version()
+ client.set_credentials(BasicLoginCredentials(admin_user, org, admin_passwd))
+ return client
+
+ except Exception as e:
+ log.info(
+ "Can't connect to a vCloud director as: {} with exception {}".format(
+ admin_user, e
+ )
+ )
+
+ def get_vim_account(self, vim_account_id: str):
+ """
+ Method to get VIM account details by its ID
+ arg - VIM ID
+ return - dict with vim account details
+ """
+ vim_account = {}
+ vim_account_info = self.common_db.get_vim_account(vim_account_id)
+
+ vim_account["name"] = vim_account_info["name"]
+ vim_account["vim_tenant_name"] = vim_account_info["vim_tenant_name"]
+ vim_account["vim_type"] = vim_account_info["vim_type"]
+ vim_account["vim_url"] = vim_account_info["vim_url"]
+ vim_account["org_user"] = vim_account_info["vim_user"]
+ vim_account["vim_uuid"] = vim_account_info["_id"]
+ if vim_account_info["_admin"]["projects_read"]:
+ vim_account["project_id"] = vim_account_info["_admin"]["projects_read"][0]
+ else:
+ vim_account["project_id"] = ""
+
+ vim_config = vim_account_info["config"]
+ vim_account["admin_username"] = vim_config["admin_username"]
+ vim_account["admin_password"] = vim_config["admin_password"]
+
+ if vim_config["orgname"] is not None:
+ vim_account["orgname"] = vim_config["orgname"]
+
+ return vim_account
+
+ def check_vim_status(self):
+ try:
+ client = self.connect_vim_as_admin()
+ if client._session:
+ org_list = client.get_org_list()
+ org_uuid = ""
+ for org in org_list.Org:
+ if org.get("name") == self.org_name:
+ org_uuid = org.get("href").split("/")[-1]
+
+ url = "{}/api/org/{}".format(self.vcloud_site, org_uuid)
+
+ headers = {
+ "Accept": "application/*+xml;version=" + API_VERSION,
+ "x-vcloud-authorization": client._session.headers[
+ "x-vcloud-authorization"
+ ],
+ }
+
+ response = requests.get(
+ url=url, headers=headers, verify=self.verify_ssl
+ )
+
+ if (
+ response.status_code != requests.codes.ok
+ ): # pylint: disable=no-member
+ log.info("check_vim_status(): failed to get org details")
+ else:
+ org_details = XmlElementTree.fromstring(response.content)
+ vdc_list = {}
+ for child in org_details:
+ if "type" in child.attrib:
+ if (
+ child.attrib["type"]
+ == "application/vnd.vmware.vcloud.vdc+xml"
+ ):
+ vdc_list[
+ child.attrib["href"].split("/")[-1:][0]
+ ] = child.attrib["name"]
+
+ if vdc_list:
+ return True
+ else:
+ return False
+ except Exception as e:
+ log.info("Exception occured while checking vim status {}".format(str(e)))
+
+ def check_vm_status(self, vapp_id):
+ try:
+ client = self.connect_vim_as_admin()
+ if client._session:
+ url = "{}/api/vApp/vapp-{}".format(self.vcloud_site, vapp_id)
+
+ headers = {
+ "Accept": "application/*+xml;version=" + API_VERSION,
+ "x-vcloud-authorization": client._session.headers[
+ "x-vcloud-authorization"
+ ],
+ }
+
+ response = requests.get(
+ url=url, headers=headers, verify=self.verify_ssl
+ )
+
+ if (
+ response.status_code != requests.codes.ok
+ ): # pylint: disable=no-member
+ log.info("check_vm_status(): failed to get vApp details")
+ else:
+ vapp_details = XmlElementTree.fromstring(response.content)
+ vm_list = []
+ for child in vapp_details:
+ if child.tag.split("}")[1] == "Children":
+ for item in child.getchildren():
+ vm_list.append(item.attrib)
+ return vm_list
+ except Exception as e:
+ log.info("Exception occured while checking vim status {}".format(str(e)))
+
+ def collect(self) -> List[Metric]:
+ metrics = []
+ vim_status = self.check_vim_status()
+ vim_account_id = self.vim_account_id
+ vim_project_id = self.vim_project_id
+ vim_tags = {"vim_account_id": vim_account_id, "project_id": vim_project_id}
+ vim_status_metric = Metric(vim_tags, "vim_status", vim_status)
+ metrics.append(vim_status_metric)
+ vnfrs = self.common_db.get_vnfrs(vim_account_id=vim_account_id)
+ if self.conf.get("collector", "vm_infra_metrics"):
+ vm_infra_metrics_enabled = str(
+ self.conf.get("collector", "vm_infra_metrics")
+ ).lower() in ("yes", "true", "1")
+ else:
+ vm_infra_metrics_enabled = True
+ if vm_infra_metrics_enabled:
+ for vnfr in vnfrs:
+ nsr_id = vnfr["nsr-id-ref"]
+ ns_name = self.common_db.get_nsr(nsr_id)["name"]
+ vnf_member_index = vnfr["member-vnf-index-ref"]
+ if vnfr["_admin"]["projects_read"]:
+ vnfr_project_id = vnfr["_admin"]["projects_read"][0]
+ else:
+ vnfr_project_id = ""
+ for vdur in vnfr["vdur"]:
+ resource_uuid = vdur["vim-id"]
+ tags = {
+ "vim_account_id": self.vim_account_id,
+ "resource_uuid": resource_uuid,
+ "nsr_id": nsr_id,
+ "ns_name": ns_name,
+ "vnf_member_index": vnf_member_index,
+ "vdur_name": vdur["name"],
+ "project_id": vnfr_project_id,
+ }
+ try:
+ vm_list = self.check_vm_status(resource_uuid)
+ for vm in vm_list:
+ if vm["status"] == "4" and vm["deployed"] == "true":
+ vm_status = 1
+ else:
+ vm_status = 0
+ vm_status_metric = Metric(tags, "vm_status", vm_status)
+ except Exception:
+ log.exception("VM status is not OK!")
+ vm_status_metric = Metric(tags, "vm_status", 0)
+ metrics.append(vm_status_metric)
+ return metrics
diff --git a/osm_mon/collector/metric.py b/osm_mon/collector/metric.py
new file mode 100644
index 0000000..741096f
--- /dev/null
+++ b/osm_mon/collector/metric.py
@@ -0,0 +1,28 @@
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+
+
+class Metric:
+ def __init__(self, tags: dict, name: str, value):
+ self.tags = tags
+ self.name = name
+ self.value = value
diff --git a/osm_mon/collector/service.py b/osm_mon/collector/service.py
new file mode 100644
index 0000000..5215e9b
--- /dev/null
+++ b/osm_mon/collector/service.py
@@ -0,0 +1,258 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+
+# This version uses a ProcessThreadPoolExecutor to limit the number of processes launched
+
+import logging
+from typing import List
+import concurrent.futures
+import time
+import keystoneauth1.exceptions
+
+from osm_mon.collector.infra_collectors.onos import OnosInfraCollector
+from osm_mon.collector.infra_collectors.openstack import OpenstackInfraCollector
+from osm_mon.collector.infra_collectors.vio import VIOInfraCollector
+from osm_mon.collector.infra_collectors.vmware import VMwareInfraCollector
+from osm_mon.collector.metric import Metric
+from osm_mon.collector.vnf_collectors.juju import VCACollector
+from osm_mon.collector.vnf_collectors.openstack import OpenstackCollector
+from osm_mon.collector.vnf_collectors.vio import VIOCollector
+from osm_mon.collector.vnf_collectors.vmware import VMwareCollector
+from osm_mon.core.common_db import CommonDbClient
+from osm_mon.core.config import Config
+
+log = logging.getLogger(__name__)
+
+VIM_COLLECTORS = {
+ "openstack": OpenstackCollector,
+ "vmware": VMwareCollector,
+ "vio": VIOCollector,
+}
+VIM_INFRA_COLLECTORS = {
+ "openstack": OpenstackInfraCollector,
+ "vmware": VMwareInfraCollector,
+ "vio": VIOInfraCollector,
+}
+SDN_INFRA_COLLECTORS = {"onosof": OnosInfraCollector, "onos_vpls": OnosInfraCollector}
+
+# Map to store vim ids and corresponding vim session objects
+vim_sess_map = {}
+
+
+# Invoked from process executor to initialize the vim session map
+def init_session(session_map: dict):
+ global vim_sess_map
+ vim_sess_map = session_map
+
+
+class CollectorService:
+ def __init__(self, config: Config):
+ self.conf = config
+ self.common_db = CommonDbClient(self.conf)
+ return
+
+ # static methods to be executed in the Processes
+ @staticmethod
+ def _get_vim_type(conf: Config, vim_account_id: str) -> str:
+ common_db = CommonDbClient(conf)
+ vim_account = common_db.get_vim_account(vim_account_id)
+ vim_type = vim_account["vim_type"]
+ if "config" in vim_account and "vim_type" in vim_account["config"]:
+ vim_type = vim_account["config"]["vim_type"].lower()
+ if vim_type == "vio" and "vrops_site" not in vim_account["config"]:
+ vim_type = "openstack"
+ return vim_type
+
+ @staticmethod
+ def _collect_vim_metrics(conf: Config, vnfr: dict, vim_account_id: str):
+ # TODO(diazb) Add support for aws
+ metrics = []
+ vim_type = CollectorService._get_vim_type(conf, vim_account_id)
+ log.debug("vim type.....{}".format(vim_type))
+ if vim_type in VIM_COLLECTORS:
+ collector = VIM_COLLECTORS[vim_type](
+ conf, vim_account_id, vim_sess_map[vim_account_id]
+ )
+ metrics = collector.collect(vnfr)
+ log.debug("Collecting vim metrics.....{}".format(metrics))
+ else:
+ log.debug("vimtype %s is not supported.", vim_type)
+ return metrics
+
+ @staticmethod
+ def _collect_vca_metrics(conf: Config, vnfr: dict):
+ metrics = []
+ vca_collector = VCACollector(conf)
+ metrics = vca_collector.collect(vnfr)
+ log.debug("Collecting vca metrics.....{}".format(metrics))
+ return metrics
+
+ @staticmethod
+ def _collect_vim_infra_metrics(conf: Config, vim_account_id: str):
+ log.info("Collecting vim infra metrics")
+ metrics = []
+ vim_type = CollectorService._get_vim_type(conf, vim_account_id)
+ if vim_type in VIM_INFRA_COLLECTORS:
+ collector = VIM_INFRA_COLLECTORS[vim_type](conf, vim_account_id)
+ metrics = collector.collect()
+ log.debug("Collecting vim infra metrics.....{}".format(metrics))
+ else:
+ log.debug("vimtype %s is not supported.", vim_type)
+ return metrics
+
+ @staticmethod
+ def _collect_sdnc_infra_metrics(conf: Config, sdnc_id: str):
+ log.info("Collecting sdnc metrics")
+ metrics = []
+ common_db = CommonDbClient(conf)
+ sdn_type = common_db.get_sdnc(sdnc_id)["type"]
+ if sdn_type in SDN_INFRA_COLLECTORS:
+ collector = SDN_INFRA_COLLECTORS[sdn_type](conf, sdnc_id)
+ metrics = collector.collect()
+ log.debug("Collecting sdnc metrics.....{}".format(metrics))
+ else:
+ log.debug("sdn_type %s is not supported.", sdn_type)
+ return metrics
+
+ @staticmethod
+ def _stop_process_pool(executor):
+ log.info("Shutting down process pool")
+ try:
+ log.debug("Stopping residual processes in the process pool")
+ for pid, process in executor._processes.items():
+ if process.is_alive():
+ process.terminate()
+ except Exception as e:
+ log.info("Exception during process termination")
+ log.debug("Exception %s" % (e))
+
+ try:
+ # Shutting down executor
+ log.debug("Shutting down process pool executor")
+ executor.shutdown()
+ except RuntimeError as e:
+ log.info("RuntimeError in shutting down executer")
+ log.debug("RuntimeError %s" % (e))
+ return
+
+ def collect_metrics(self) -> List[Metric]:
+ vnfrs = self.common_db.get_vnfrs()
+ metrics = []
+
+ # Get all vim ids regiestered in osm and create their corresponding vim session objects
+ # Vim ids and their corresponding session objects are stored in vim-session-map
+ # It optimizes the number of authentication tokens created in vim for metric colleciton
+ vim_sess_map.clear()
+ vims = self.common_db.get_vim_accounts()
+ for vim in vims:
+ vim_type = CollectorService._get_vim_type(self.conf, vim["_id"])
+ if vim_type in VIM_INFRA_COLLECTORS:
+ collector = VIM_INFRA_COLLECTORS[vim_type](self.conf, vim["_id"])
+ vim_sess = collector.vim_session if vim_type == "openstack" else None
+ # Populate the vim session map with vim ids and corresponding session objects
+ # vim session objects are stopred only for vim type openstack
+ if vim_sess:
+ vim_sess_map[vim["_id"]] = vim_sess
+
+ start_time = time.time()
+ # Starting executor pool with pool size process_pool_size. Default process_pool_size is 20
+ # init_session is called to assign the session map to the gloabal vim session map variable
+ with concurrent.futures.ProcessPoolExecutor(
+ self.conf.get("collector", "process_pool_size"),
+ initializer=init_session,
+ initargs=(vim_sess_map,),
+ ) as executor:
+ log.info(
+ "Started metric collector process pool with pool size %s"
+ % (self.conf.get("collector", "process_pool_size"))
+ )
+ futures = []
+ for vnfr in vnfrs:
+ nsr_id = vnfr["nsr-id-ref"]
+ vnf_member_index = vnfr["member-vnf-index-ref"]
+ vim_account_id = self.common_db.get_vim_account_id(
+ nsr_id, vnf_member_index
+ )
+ futures.append(
+ executor.submit(
+ CollectorService._collect_vim_metrics,
+ self.conf,
+ vnfr,
+ vim_account_id,
+ )
+ )
+ futures.append(
+ executor.submit(
+ CollectorService._collect_vca_metrics, self.conf, vnfr
+ )
+ )
+
+ for vim in vims:
+ futures.append(
+ executor.submit(
+ CollectorService._collect_vim_infra_metrics,
+ self.conf,
+ vim["_id"],
+ )
+ )
+
+ sdncs = self.common_db.get_sdncs()
+ for sdnc in sdncs:
+ futures.append(
+ executor.submit(
+ CollectorService._collect_sdnc_infra_metrics,
+ self.conf,
+ sdnc["_id"],
+ )
+ )
+
+ try:
+ # Wait for future calls to complete till process_execution_timeout. Default is 50 seconds
+ for future in concurrent.futures.as_completed(
+ futures, self.conf.get("collector", "process_execution_timeout")
+ ):
+ try:
+ result = future.result(
+ timeout=int(
+ self.conf.get("collector", "process_execution_timeout")
+ )
+ )
+ metrics.extend(result)
+ log.debug("result = %s" % (result))
+ except keystoneauth1.exceptions.connection.ConnectionError as e:
+ log.info("Keystone connection error during metric collection")
+ log.debug("Keystone connection error exception %s" % (e))
+ except concurrent.futures.TimeoutError as e:
+ # Some processes have not completed due to timeout error
+ log.info(
+ "Some processes have not finished due to TimeoutError exception"
+ )
+ log.debug("concurrent.futures.TimeoutError exception %s" % (e))
+
+ # Shutting down process pool executor
+ CollectorService._stop_process_pool(executor)
+
+ end_time = time.time()
+ log.info("Collection completed in %s seconds", end_time - start_time)
+
+ return metrics
diff --git a/osm_mon/collector/utils/__init__.py b/osm_mon/collector/utils/__init__.py
new file mode 100644
index 0000000..cbff444
--- /dev/null
+++ b/osm_mon/collector/utils/__init__.py
@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2019 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
diff --git a/osm_mon/collector/utils/openstack.py b/osm_mon/collector/utils/openstack.py
new file mode 100644
index 0000000..89b13d1
--- /dev/null
+++ b/osm_mon/collector/utils/openstack.py
@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2019 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+import logging
+from os import makedirs, path
+
+from keystoneauth1 import session
+from keystoneauth1.identity import v3
+
+from osm_mon.core.exceptions import CertificateNotCreated
+
+log = logging.getLogger(__name__)
+
+
+class OpenstackUtils:
+ @staticmethod
+ def get_session(creds: dict):
+ verify_ssl = True
+ project_domain_name = "Default"
+ user_domain_name = "Default"
+ try:
+ if "config" in creds:
+ vim_config = creds["config"]
+ if "insecure" in vim_config and vim_config["insecure"]:
+ verify_ssl = False
+ if "ca_cert" in vim_config:
+ verify_ssl = vim_config["ca_cert"]
+ elif "ca_cert_content" in vim_config:
+ vim_config = OpenstackUtils._create_file_cert(
+ vim_config, creds["_id"]
+ )
+ verify_ssl = vim_config["ca_cert"]
+ if "project_domain_name" in vim_config:
+ project_domain_name = vim_config["project_domain_name"]
+ if "user_domain_name" in vim_config:
+ user_domain_name = vim_config["user_domain_name"]
+ auth = v3.Password(
+ auth_url=creds["vim_url"],
+ username=creds["vim_user"],
+ password=creds["vim_password"],
+ project_name=creds["vim_tenant_name"],
+ project_domain_name=project_domain_name,
+ user_domain_name=user_domain_name,
+ )
+ return session.Session(auth=auth, verify=verify_ssl, timeout=10)
+ except CertificateNotCreated as e:
+ log.error(e)
+
+ @staticmethod
+ def _create_file_cert(vim_config: dict, target_id: str) -> dict:
+ """
+ Process vim config, creating vim configuration files as ca_cert
+ Creates a folder '/app/osm_mon/certs/{target_id}' and the ca_cert inside
+ :param target_id: vim-id
+ :param db_vim: Vim dictionary obtained from database
+ :return: Modified vim configuration dictionary.
+ """
+
+ work_dir = f"/app/osm_mon/certs/{target_id}"
+ file_name = ""
+
+ try:
+ if vim_config.get("ca_cert_content"):
+ if not path.isdir(work_dir):
+ makedirs(work_dir)
+
+ file_name = f"{work_dir}/ca_cert"
+ with open(file_name, "w") as f:
+ f.write(vim_config["ca_cert_content"])
+ del vim_config["ca_cert_content"]
+ vim_config["ca_cert"] = file_name
+ return vim_config
+ except Exception as e:
+ if file_name:
+ raise CertificateNotCreated(f"Error writing to file '{file_name}': {e}")
+ else:
+ raise CertificateNotCreated(
+ f"Error creating the directory '{work_dir}': {e}"
+ )
diff --git a/osm_mon/collector/vnf_collectors/__init__.py b/osm_mon/collector/vnf_collectors/__init__.py
new file mode 100644
index 0000000..971f4e9
--- /dev/null
+++ b/osm_mon/collector/vnf_collectors/__init__.py
@@ -0,0 +1,21 @@
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
diff --git a/osm_mon/collector/vnf_collectors/base.py b/osm_mon/collector/vnf_collectors/base.py
new file mode 100644
index 0000000..0ec12b5
--- /dev/null
+++ b/osm_mon/collector/vnf_collectors/base.py
@@ -0,0 +1,33 @@
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+from typing import List
+
+from osm_mon.collector.metric import Metric
+from osm_mon.core.config import Config
+
+
+class BaseCollector:
+ def __init__(self, config: Config):
+ pass
+
+ def collect(self, vnfr: dict) -> List[Metric]:
+ pass
diff --git a/osm_mon/collector/vnf_collectors/base_vim.py b/osm_mon/collector/vnf_collectors/base_vim.py
new file mode 100644
index 0000000..6c270f4
--- /dev/null
+++ b/osm_mon/collector/vnf_collectors/base_vim.py
@@ -0,0 +1,29 @@
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+
+from osm_mon.collector.vnf_collectors.base import BaseCollector
+from osm_mon.core.config import Config
+
+
+class BaseVimCollector(BaseCollector):
+ def __init__(self, config: Config, vim_account_id: str):
+ super().__init__(config)
diff --git a/osm_mon/collector/vnf_collectors/juju.py b/osm_mon/collector/vnf_collectors/juju.py
new file mode 100644
index 0000000..fbc6bc2
--- /dev/null
+++ b/osm_mon/collector/vnf_collectors/juju.py
@@ -0,0 +1,157 @@
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+import asyncio
+import logging
+from typing import List
+
+from n2vc.n2vc_juju_conn import N2VCJujuConnector
+
+from osm_mon.collector.metric import Metric
+from osm_mon.collector.vnf_collectors.base import BaseCollector
+from osm_mon.collector.vnf_metric import VnfMetric
+from osm_mon.core.common_db import CommonDbClient
+from osm_mon.core.config import Config
+from osm_mon.core.exceptions import VcaDeploymentInfoNotFound
+
+log = logging.getLogger(__name__)
+
+
+class VCACollector(BaseCollector):
+ def __init__(self, config: Config):
+ super().__init__(config)
+ self.common_db = CommonDbClient(config)
+ # host = config.get("vca", "host")
+ # port = config.get("vca", "port") if "port" in config.conf["vca"] else 17070
+
+ # Backwards compatibility
+ if "cacert" in config.conf["vca"]:
+ ca_cert = config.conf["vca"].pop("cacert")
+ config.set("vca", "ca_cert", ca_cert)
+
+ if "pubkey" in config.conf["vca"]:
+ public_key = config.conf["vca"].pop("pubkey")
+ config.set("vca", "public_key", public_key)
+
+ if "apiproxy" in config.conf["vca"]:
+ api_proxy = config.conf["vca"].pop("apiproxy")
+ config.set("vca", "api_proxy", api_proxy)
+
+ self.n2vc = N2VCJujuConnector(
+ db=self.common_db.common_db,
+ fs=object(),
+ log=log,
+ on_update_db=None,
+ )
+
+ def collect(self, vnfr: dict) -> List[Metric]:
+ nsr_id = vnfr["nsr-id-ref"]
+ vnf_member_index = vnfr["member-vnf-index-ref"]
+ vnfd = self.common_db.get_vnfd(vnfr["vnfd-id"])
+
+ # Populate extra tags for metrics
+ tags = {}
+ tags["ns_name"] = self.common_db.get_nsr(nsr_id)["name"]
+ if vnfr["_admin"]["projects_read"]:
+ tags["project_id"] = vnfr["_admin"]["projects_read"][0]
+ else:
+ tags["project_id"] = ""
+
+ metrics = []
+ vdur = None
+ lcm_ops = vnfd["df"][0].get("lcm-operations-configuration")
+ if not lcm_ops:
+ return metrics
+ ops_config = lcm_ops.get("operate-vnf-op-config")
+ if not ops_config:
+ return metrics
+ day12ops = ops_config.get("day1-2", [])
+ for day12op in day12ops:
+ if day12op and "metrics" in day12op:
+ vdur = next(
+ filter(
+ lambda vdur: vdur["vdu-id-ref"] == day12op["id"], vnfr["vdur"]
+ )
+ )
+
+ # This avoids errors when vdur records have not been completely filled
+ if vdur and "name" in vdur:
+ try:
+ vca_deployment_info = self.get_vca_deployment_info(
+ nsr_id,
+ vnf_member_index,
+ vdur["vdu-id-ref"],
+ vdur["count-index"],
+ )
+ except VcaDeploymentInfoNotFound as e:
+ log.warning(repr(e))
+ continue
+ # This avoids errors before application and model is not ready till they are occured
+ if vca_deployment_info.get("model") and vca_deployment_info.get(
+ "application"
+ ):
+ measures = asyncio.run(
+ self.n2vc.get_metrics(
+ vca_deployment_info["model"],
+ vca_deployment_info["application"],
+ vca_id=vnfr.get("vca-id"),
+ )
+ )
+ log.debug("Measures: %s", measures)
+ for measure_list in measures.values():
+ for measure in measure_list:
+ log.debug("Measure: %s", measure)
+ metric = VnfMetric(
+ nsr_id,
+ vnf_member_index,
+ vdur["name"],
+ measure["key"],
+ float(measure["value"]),
+ tags,
+ )
+ metrics.append(metric)
+
+ return metrics
+
+ def get_vca_deployment_info(
+ self, nsr_id, vnf_member_index, vdu_id=None, vdu_count=0
+ ):
+ nsr = self.common_db.get_nsr(nsr_id)
+ for vca_deployment in nsr["_admin"]["deployed"]["VCA"]:
+ if vca_deployment:
+ if vdu_id is None:
+ if (
+ vca_deployment["member-vnf-index"] == vnf_member_index
+ and vca_deployment["vdu_id"] is None
+ ):
+ return vca_deployment
+ else:
+ if (
+ vca_deployment["member-vnf-index"] == vnf_member_index
+ and vca_deployment["vdu_id"] == vdu_id
+ and vca_deployment["vdu_count_index"] == vdu_count
+ ):
+ return vca_deployment
+ raise VcaDeploymentInfoNotFound(
+ "VCA deployment info for nsr_id {}, index {}, vdu_id {} and vdu_count_index {} not found.".format(
+ nsr_id, vnf_member_index, vdu_id, vdu_count
+ )
+ )
diff --git a/osm_mon/collector/vnf_collectors/openstack.py b/osm_mon/collector/vnf_collectors/openstack.py
new file mode 100644
index 0000000..6aedf88
--- /dev/null
+++ b/osm_mon/collector/vnf_collectors/openstack.py
@@ -0,0 +1,458 @@
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+from enum import Enum
+import logging
+import time
+from typing import List
+
+from ceilometerclient import client as ceilometer_client
+from ceilometerclient.exc import HTTPException
+import gnocchiclient.exceptions
+from gnocchiclient.v1 import client as gnocchi_client
+from keystoneauth1.exceptions.catalog import EndpointNotFound
+from keystoneclient.v3 import client as keystone_client
+from neutronclient.v2_0 import client as neutron_client
+from prometheus_api_client import PrometheusConnect as prometheus_client
+
+from osm_mon.collector.metric import Metric
+from osm_mon.collector.utils.openstack import OpenstackUtils
+from osm_mon.collector.vnf_collectors.base_vim import BaseVimCollector
+from osm_mon.collector.vnf_metric import VnfMetric
+from osm_mon.core.common_db import CommonDbClient
+from osm_mon.core.config import Config
+
+
+log = logging.getLogger(__name__)
+
+METRIC_MAPPINGS = {
+ "average_memory_utilization": "memory.usage",
+ "disk_read_ops": "disk.read.requests.rate",
+ "disk_write_ops": "disk.write.requests.rate",
+ "disk_read_bytes": "disk.read.bytes.rate",
+ "disk_write_bytes": "disk.write.bytes.rate",
+ "packets_in_dropped": "network.outgoing.packets.drop",
+ "packets_out_dropped": "network.incoming.packets.drop",
+ "packets_received": "network.incoming.packets.rate",
+ "packets_sent": "network.outgoing.packets.rate",
+ "cpu_utilization": "cpu",
+}
+
+METRIC_MAPPINGS_FOR_PROMETHEUS_TSBD = {
+ "cpu_utilization": "cpu",
+ "average_memory_utilization": "memory_usage",
+ "disk_read_ops": "disk_device_read_requests",
+ "disk_write_ops": "disk_device_write_requests",
+ "disk_read_bytes": "disk_device_read_bytes",
+ "disk_write_bytes": "disk_device_write_bytes",
+ "packets_in_dropped": "network_incoming_packets_drop",
+ "packets_out_dropped": "network_outgoing_packets_drop",
+ "packets_received": "network_incoming_packets",
+ "packets_sent": "network_outgoing_packets",
+}
+
+# Metrics which have new names in Rocky and higher releases
+METRIC_MAPPINGS_FOR_ROCKY_AND_NEWER_RELEASES = {
+ "disk_read_ops": "disk.device.read.requests",
+ "disk_write_ops": "disk.device.write.requests",
+ "disk_read_bytes": "disk.device.read.bytes",
+ "disk_write_bytes": "disk.device.write.bytes",
+ "packets_received": "network.incoming.packets",
+ "packets_sent": "network.outgoing.packets",
+}
+
+METRIC_MULTIPLIERS = {"cpu": 0.0000001}
+
+METRIC_AGGREGATORS = {"cpu": "rate:mean"}
+
+INTERFACE_METRICS = [
+ "packets_in_dropped",
+ "packets_out_dropped",
+ "packets_received",
+ "packets_sent",
+]
+
+INSTANCE_DISK = [
+ "disk_read_ops",
+ "disk_write_ops",
+ "disk_read_bytes",
+ "disk_write_bytes",
+]
+
+
+class MetricType(Enum):
+ INSTANCE = "instance"
+ INTERFACE_ALL = "interface_all"
+ INTERFACE_ONE = "interface_one"
+ INSTANCEDISK = "instancedisk"
+
+
+class OpenstackCollector(BaseVimCollector):
+ def __init__(self, config: Config, vim_account_id: str, vim_session: object):
+ super().__init__(config, vim_account_id)
+ self.common_db = CommonDbClient(config)
+ vim_account = self.common_db.get_vim_account(vim_account_id)
+ self.backend = self._get_backend(vim_account, vim_session)
+
+ def _build_keystone_client(self, vim_account: dict) -> keystone_client.Client:
+ sess = OpenstackUtils.get_session(vim_account)
+ return keystone_client.Client(session=sess)
+
+ def _get_resource_uuid(
+ self, nsr_id: str, vnf_member_index: str, vdur_name: str
+ ) -> str:
+ vdur = self.common_db.get_vdur(nsr_id, vnf_member_index, vdur_name)
+ return vdur["vim-id"]
+
+ def collect(self, vnfr: dict) -> List[Metric]:
+ nsr_id = vnfr["nsr-id-ref"]
+ vnf_member_index = vnfr["member-vnf-index-ref"]
+ vnfd = self.common_db.get_vnfd(vnfr["vnfd-id"])
+ # Populate extra tags for metrics
+ tags = {}
+ tags["ns_name"] = self.common_db.get_nsr(nsr_id)["name"]
+ if vnfr["_admin"]["projects_read"]:
+ tags["project_id"] = vnfr["_admin"]["projects_read"][0]
+ else:
+ tags["project_id"] = ""
+
+ metrics = []
+
+ for vdur in vnfr["vdur"]:
+ # This avoids errors when vdur records have not been completely filled
+ if "name" not in vdur:
+ continue
+ vdu = next(filter(lambda vdu: vdu["id"] == vdur["vdu-id-ref"], vnfd["vdu"]))
+ if "monitoring-parameter" in vdu:
+ for param in vdu["monitoring-parameter"]:
+ metric_name = param["performance-metric"]
+ log.info(f"Using an {type(self.backend)} as backend")
+ if type(self.backend) is PrometheusTSBDBackend:
+ openstack_metric_name = self.backend.map_metric(metric_name)
+ else:
+ try:
+ openstack_metric_name = METRIC_MAPPINGS[metric_name]
+ except KeyError:
+ continue
+ metric_type = self._get_metric_type(metric_name)
+ try:
+ resource_id = self._get_resource_uuid(
+ nsr_id, vnf_member_index, vdur["name"]
+ )
+ except ValueError:
+ log.warning(
+ "Could not find resource_uuid for vdur %s, vnf_member_index %s, nsr_id %s. "
+ "Was it recently deleted?",
+ vdur["name"],
+ vnf_member_index,
+ nsr_id,
+ )
+ continue
+ try:
+ log.info(
+ "Collecting metric type: %s and metric_name: %s and resource_id %s and ",
+ metric_type,
+ metric_name,
+ resource_id,
+ )
+ value = self.backend.collect_metric(
+ metric_type, openstack_metric_name, resource_id
+ )
+
+ if (
+ value is None
+ and metric_name
+ in METRIC_MAPPINGS_FOR_ROCKY_AND_NEWER_RELEASES
+ and type(self.backend) is not PrometheusTSBDBackend
+ ):
+ # Reattempting metric collection with new metric names.
+ # Some metric names have changed in newer Openstack releases
+ log.info(
+ "Reattempting metric collection for type: %s and name: %s and resource_id %s",
+ metric_type,
+ metric_name,
+ resource_id,
+ )
+ openstack_metric_name = (
+ METRIC_MAPPINGS_FOR_ROCKY_AND_NEWER_RELEASES[
+ metric_name
+ ]
+ )
+ value = self.backend.collect_metric(
+ metric_type, openstack_metric_name, resource_id
+ )
+ if value is not None:
+ log.info("value: %s", value)
+ metric = VnfMetric(
+ nsr_id,
+ vnf_member_index,
+ vdur["name"],
+ metric_name,
+ value,
+ tags,
+ )
+ metrics.append(metric)
+ else:
+ log.info("metric value is empty")
+ except Exception as e:
+ log.exception(
+ "Error collecting metric %s for vdu %s"
+ % (metric_name, vdur["name"])
+ )
+ log.info("Error in metric collection: %s" % e)
+ return metrics
+
+ def _get_backend(self, vim_account: dict, vim_session: object):
+ if vim_account.get("prometheus-config"):
+ try:
+ tsbd = PrometheusTSBDBackend(vim_account)
+ log.debug("Using prometheustsbd backend to collect metric")
+ return tsbd
+ except Exception as e:
+ log.error(f"Can't create prometheus client, {e}")
+ return None
+ try:
+ gnocchi = GnocchiBackend(vim_account, vim_session)
+ gnocchi.client.metric.list(limit=1)
+ log.debug("Using gnocchi backend to collect metric")
+ return gnocchi
+ except (HTTPException, EndpointNotFound):
+ ceilometer = CeilometerBackend(vim_account, vim_session)
+ ceilometer.client.capabilities.get()
+ log.debug("Using ceilometer backend to collect metric")
+ return ceilometer
+
+ def _get_metric_type(self, metric_name: str) -> MetricType:
+ if metric_name not in INTERFACE_METRICS:
+ if metric_name not in INSTANCE_DISK:
+ return MetricType.INSTANCE
+ else:
+ return MetricType.INSTANCEDISK
+ else:
+ return MetricType.INTERFACE_ALL
+
+
+class OpenstackBackend:
+ def collect_metric(
+ self, metric_type: MetricType, metric_name: str, resource_id: str
+ ):
+ pass
+
+
+class PrometheusTSBDBackend(OpenstackBackend):
+ def __init__(self, vim_account: dict):
+ self.map = self._build_map(vim_account)
+ self.cred = vim_account["prometheus-config"].get("prometheus-cred")
+ self.client = self._build_prometheus_client(
+ vim_account["prometheus-config"]["prometheus-url"]
+ )
+
+ def _build_prometheus_client(self, url: str) -> prometheus_client:
+ return prometheus_client(url, disable_ssl=True)
+
+ def _build_map(self, vim_account: dict) -> dict:
+ custom_map = METRIC_MAPPINGS_FOR_PROMETHEUS_TSBD
+ if "prometheus-map" in vim_account["prometheus-config"]:
+ custom_map.update(vim_account["prometheus-config"]["prometheus-map"])
+ return custom_map
+
+ def collect_metric(
+ self, metric_type: MetricType, metric_name: str, resource_id: str
+ ):
+ metric = self.query_metric(metric_name, resource_id)
+ return metric["value"][1] if metric else None
+
+ def map_metric(self, metric_name: str):
+ return self.map[metric_name]
+
+ def query_metric(self, metric_name, resource_id=None):
+ metrics = self.client.get_current_metric_value(metric_name=metric_name)
+ if resource_id:
+ metric = next(
+ filter(lambda x: resource_id in x["metric"]["resource_id"], metrics)
+ )
+ return metric
+ return metrics
+
+
+class GnocchiBackend(OpenstackBackend):
+ def __init__(self, vim_account: dict, vim_session: object):
+ self.client = self._build_gnocchi_client(vim_account, vim_session)
+ self.neutron = self._build_neutron_client(vim_account, vim_session)
+
+ def _build_gnocchi_client(
+ self, vim_account: dict, vim_session: object
+ ) -> gnocchi_client.Client:
+ return gnocchi_client.Client(session=vim_session)
+
+ def _build_neutron_client(
+ self, vim_account: dict, vim_session: object
+ ) -> neutron_client.Client:
+ return neutron_client.Client(session=vim_session)
+
+ def collect_metric(
+ self, metric_type: MetricType, metric_name: str, resource_id: str
+ ):
+ if metric_type == MetricType.INTERFACE_ALL:
+ return self._collect_interface_all_metric(metric_name, resource_id)
+
+ elif metric_type == MetricType.INSTANCE:
+ return self._collect_instance_metric(metric_name, resource_id)
+
+ elif metric_type == MetricType.INSTANCEDISK:
+ return self._collect_instance_disk_metric(metric_name, resource_id)
+
+ else:
+ raise Exception("Unknown metric type %s" % metric_type.value)
+
+ def _collect_interface_all_metric(self, openstack_metric_name, resource_id):
+ total_measure = None
+ interfaces = self.client.resource.search(
+ resource_type="instance_network_interface",
+ query={"=": {"instance_id": resource_id}},
+ )
+ for interface in interfaces:
+ try:
+ measures = self.client.metric.get_measures(
+ openstack_metric_name, resource_id=interface["id"], limit=1
+ )
+ if measures:
+ if not total_measure:
+ total_measure = 0.0
+ total_measure += measures[-1][2]
+ except (gnocchiclient.exceptions.NotFound, TypeError) as e:
+ # Gnocchi in some Openstack versions raise TypeError instead of NotFound
+ log.debug(
+ "No metric %s found for interface %s: %s",
+ openstack_metric_name,
+ interface["id"],
+ e,
+ )
+ return total_measure
+
+ def _collect_instance_disk_metric(self, openstack_metric_name, resource_id):
+ value = None
+ instances = self.client.resource.search(
+ resource_type="instance_disk",
+ query={"=": {"instance_id": resource_id}},
+ )
+ for instance in instances:
+ try:
+ measures = self.client.metric.get_measures(
+ openstack_metric_name, resource_id=instance["id"], limit=1
+ )
+ if measures:
+ value = measures[-1][2]
+
+ except gnocchiclient.exceptions.NotFound as e:
+ log.debug(
+ "No metric %s found for instance disk %s: %s",
+ openstack_metric_name,
+ instance["id"],
+ e,
+ )
+ return value
+
+ def _collect_instance_metric(self, openstack_metric_name, resource_id):
+ value = None
+ try:
+ aggregation = METRIC_AGGREGATORS.get(openstack_metric_name)
+
+ try:
+ measures = self.client.metric.get_measures(
+ openstack_metric_name,
+ aggregation=aggregation,
+ start=time.time() - 1200,
+ resource_id=resource_id,
+ )
+ if measures:
+ value = measures[-1][2]
+ except (
+ gnocchiclient.exceptions.NotFound,
+ gnocchiclient.exceptions.BadRequest,
+ TypeError,
+ ) as e:
+ # CPU metric in previous Openstack versions do not support rate:mean aggregation method
+ # Gnocchi in some Openstack versions raise TypeError instead of NotFound or BadRequest
+ if openstack_metric_name == "cpu":
+ log.debug(
+ "No metric %s found for instance %s: %s",
+ openstack_metric_name,
+ resource_id,
+ e,
+ )
+ log.info(
+ "Retrying to get metric %s for instance %s without aggregation",
+ openstack_metric_name,
+ resource_id,
+ )
+ measures = self.client.metric.get_measures(
+ openstack_metric_name, resource_id=resource_id, limit=1
+ )
+ else:
+ raise e
+ # measures[-1] is the last measure
+ # measures[-2] is the previous measure
+ # measures[x][2] is the value of the metric
+ if measures and len(measures) >= 2:
+ value = measures[-1][2] - measures[-2][2]
+ if value:
+ # measures[-1][0] is the time of the reporting interval
+ # measures[-1][1] is the duration of the reporting interval
+ if aggregation:
+ # If this is an aggregate, we need to divide the total over the reported time period.
+ # Even if the aggregation method is not supported by Openstack, the code will execute it
+ # because aggregation is specified in METRIC_AGGREGATORS
+ value = value / measures[-1][1]
+ if openstack_metric_name in METRIC_MULTIPLIERS:
+ value = value * METRIC_MULTIPLIERS[openstack_metric_name]
+ except gnocchiclient.exceptions.NotFound as e:
+ log.debug(
+ "No metric %s found for instance %s: %s",
+ openstack_metric_name,
+ resource_id,
+ e,
+ )
+ return value
+
+
+class CeilometerBackend(OpenstackBackend):
+ def __init__(self, vim_account: dict, vim_session: object):
+ self.client = self._build_ceilometer_client(vim_account, vim_session)
+
+ def _build_ceilometer_client(
+ self, vim_account: dict, vim_session: object
+ ) -> ceilometer_client.Client:
+ return ceilometer_client.Client("2", session=vim_session)
+
+ def collect_metric(
+ self, metric_type: MetricType, metric_name: str, resource_id: str
+ ):
+ if metric_type != MetricType.INSTANCE:
+ raise NotImplementedError(
+ "Ceilometer backend only support instance metrics"
+ )
+ measures = self.client.samples.list(
+ meter_name=metric_name,
+ limit=1,
+ q=[{"field": "resource_id", "op": "eq", "value": resource_id}],
+ )
+ return measures[0].counter_volume if measures else None
diff --git a/osm_mon/collector/vnf_collectors/vio.py b/osm_mon/collector/vnf_collectors/vio.py
new file mode 100644
index 0000000..130b253
--- /dev/null
+++ b/osm_mon/collector/vnf_collectors/vio.py
@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+
+##
+# Copyright 2016-2017 VMware Inc.
+# This file is part of ETSI OSM
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: osslegalrouting@vmware.com
+##
+
+import logging
+
+from osm_mon.collector.vnf_collectors.base_vim import BaseVimCollector
+from osm_mon.collector.vnf_collectors.vrops.vrops_helper import vROPS_Helper
+from osm_mon.core.common_db import CommonDbClient
+from osm_mon.core.config import Config
+
+log = logging.getLogger(__name__)
+
+
+class VIOCollector(BaseVimCollector):
+ def __init__(self, config: Config, vim_account_id: str, vim_session: object):
+ super().__init__(config, vim_account_id)
+ self.common_db = CommonDbClient(config)
+ cfg = self.get_vim_account(vim_account_id)
+ self.vrops = vROPS_Helper(
+ vrops_site=cfg["vrops_site"],
+ vrops_user=cfg["vrops_user"],
+ vrops_password=cfg["vrops_password"],
+ )
+
+ def get_vim_account(self, vim_account_id: str):
+ vim_account_info = self.common_db.get_vim_account(vim_account_id)
+ return vim_account_info["config"]
+
+ def collect(self, vnfr: dict):
+ vnfd = self.common_db.get_vnfd(vnfr["vnfd-id"])
+ vdu_mappings = {}
+
+ # Populate extra tags for metrics
+ nsr_id = vnfr["nsr-id-ref"]
+ tags = {}
+ tags["ns_name"] = self.common_db.get_nsr(nsr_id)["name"]
+ if vnfr["_admin"]["projects_read"]:
+ tags["project_id"] = vnfr["_admin"]["projects_read"][0]
+ else:
+ tags["project_id"] = ""
+
+ # Fetch the list of all known resources from vROPS.
+ resource_list = self.vrops.get_vm_resource_list_from_vrops()
+
+ for vdur in vnfr["vdur"]:
+ # This avoids errors when vdur records have not been completely filled
+ if "name" not in vdur:
+ continue
+
+ vdu = next(filter(lambda vdu: vdu["id"] == vdur["vdu-id-ref"], vnfd["vdu"]))
+ if "monitoring-parameter" not in vdu:
+ continue
+
+ vim_id = vdur["vim-id"]
+ vdu_mappings[vim_id] = {"name": vdur["name"]}
+
+ # Map the vROPS instance id to the vim-id so we can look it up.
+ for resource in resource_list:
+ for resourceIdentifier in resource["resourceKey"][
+ "resourceIdentifiers"
+ ]:
+ if (
+ resourceIdentifier["identifierType"]["name"]
+ == "VMEntityInstanceUUID"
+ ):
+ if resourceIdentifier["value"] != vim_id:
+ continue
+ vdu_mappings[vim_id]["vrops_id"] = resource["identifier"]
+
+ if len(vdu_mappings) != 0:
+ return self.vrops.get_metrics(
+ vdu_mappings=vdu_mappings,
+ monitoring_params=vdu["monitoring-parameter"],
+ vnfr=vnfr,
+ tags=tags,
+ )
+ else:
+ return []
diff --git a/osm_mon/collector/vnf_collectors/vmware.py b/osm_mon/collector/vnf_collectors/vmware.py
new file mode 100644
index 0000000..7284298
--- /dev/null
+++ b/osm_mon/collector/vnf_collectors/vmware.py
@@ -0,0 +1,269 @@
+# -*- coding: utf-8 -*-
+
+##
+# Copyright 2016-2019 VMware Inc.
+# This file is part of ETSI OSM
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: osslegalrouting@vmware.com
+##
+
+import logging
+import traceback
+from lxml import etree as XmlElementTree
+
+import requests
+from pyvcloud.vcd.client import BasicLoginCredentials
+from pyvcloud.vcd.client import Client
+
+from osm_mon.collector.vnf_collectors.base_vim import BaseVimCollector
+from osm_mon.collector.vnf_collectors.vrops.vrops_helper import vROPS_Helper
+from osm_mon.core.common_db import CommonDbClient
+from osm_mon.core.config import Config
+
+log = logging.getLogger(__name__)
+
+API_VERSION = "27.0"
+
+
+class VMwareCollector(BaseVimCollector):
+ def __init__(self, config: Config, vim_account_id: str, vim_session: object):
+ super().__init__(config, vim_account_id)
+ self.common_db = CommonDbClient(config)
+ vim_account = self.get_vim_account(vim_account_id)
+ self.vcloud_site = vim_account["vim_url"]
+ self.admin_username = vim_account["admin_username"]
+ self.admin_password = vim_account["admin_password"]
+ self.vrops = vROPS_Helper(
+ vrops_site=vim_account["vrops_site"],
+ vrops_user=vim_account["vrops_user"],
+ vrops_password=vim_account["vrops_password"],
+ )
+
+ def connect_as_admin(self):
+ """Method connect as pvdc admin user to vCloud director.
+ There are certain action that can be done only by provider vdc admin user.
+ Organization creation / provider network creation etc.
+
+ Returns:
+ The return client object that letter can be used to connect to vcloud direct as admin for provider vdc
+ """
+
+ log.debug("Logging into vCD org as admin.")
+
+ admin_user = None
+ try:
+ host = self.vcloud_site
+ admin_user = self.admin_username
+ admin_passwd = self.admin_password
+ org = "System"
+ client = Client(host, verify_ssl_certs=False)
+ client.set_highest_supported_version()
+ client.set_credentials(BasicLoginCredentials(admin_user, org, admin_passwd))
+ return client
+
+ except Exception as e:
+ log.error(
+ "Can't connect to a vCloud director as: {} with exception {}".format(
+ admin_user, e
+ )
+ )
+
+ def get_vim_account(self, vim_account_id: str):
+ """
+ Method to get VIM account details by its ID
+ arg - VIM ID
+ return - dict with vim account details
+ """
+ vim_account = {}
+ vim_account_info = self.common_db.get_vim_account(vim_account_id)
+
+ vim_account["vim_url"] = vim_account_info["vim_url"]
+
+ vim_config = vim_account_info["config"]
+ vim_account["admin_username"] = vim_config["admin_username"]
+ vim_account["admin_password"] = vim_config["admin_password"]
+ vim_account["vrops_site"] = vim_config["vrops_site"]
+ vim_account["vrops_user"] = vim_config["vrops_user"]
+ vim_account["vrops_password"] = vim_config["vrops_password"]
+
+ return vim_account
+
+ def get_vm_moref_id(self, vapp_uuid):
+ """
+ Method to get the moref_id of given VM
+ arg - vapp_uuid
+ return - VM mored_id
+ """
+ vm_moref_id = None
+ try:
+ if vapp_uuid:
+ vm_details = self.get_vapp_details_rest(vapp_uuid)
+
+ if vm_details and "vm_vcenter_info" in vm_details:
+ vm_moref_id = vm_details["vm_vcenter_info"].get("vm_moref_id", None)
+ log.debug(
+ "Found vm_moref_id: {} for vApp UUID: {}".format(
+ vm_moref_id, vapp_uuid
+ )
+ )
+ else:
+ log.error(
+ "Failed to find vm_moref_id from vApp UUID: {}".format(
+ vapp_uuid
+ )
+ )
+
+ except Exception as exp:
+ log.warning(
+ "Error occurred while getting VM moref ID for VM: {}\n{}".format(
+ exp, traceback.format_exc()
+ )
+ )
+
+ return vm_moref_id
+
+ def get_vapp_details_rest(self, vapp_uuid=None):
+ """
+ Method retrieve vapp detail from vCloud director
+ vapp_uuid - is vapp identifier.
+ Returns - VM MOref ID or return None
+ """
+ parsed_respond = {}
+
+ if vapp_uuid is None:
+ return parsed_respond
+
+ vca = self.connect_as_admin()
+
+ if not vca:
+ log.error("Failed to connect to vCD")
+ return parsed_respond
+
+ url_list = [self.vcloud_site, "/api/vApp/vapp-", vapp_uuid]
+ get_vapp_restcall = "".join(url_list)
+
+ if vca._session:
+ headers = {
+ "Accept": "application/*+xml;version=" + API_VERSION,
+ "x-vcloud-authorization": vca._session.headers[
+ "x-vcloud-authorization"
+ ],
+ }
+ response = requests.get(get_vapp_restcall, headers=headers)
+
+ if response.status_code != 200:
+ log.error(
+ "REST API call {} failed. Return status code {}".format(
+ get_vapp_restcall, response.content
+ )
+ )
+ return parsed_respond
+
+ try:
+ xmlroot_respond = XmlElementTree.fromstring(response.content)
+
+ namespaces = {
+ "vm": "http://www.vmware.com/vcloud/v1.5",
+ "vmext": "http://www.vmware.com/vcloud/extension/v1.5",
+ "xmlns": "http://www.vmware.com/vcloud/v1.5",
+ }
+
+ # parse children section for other attrib
+ children_section = xmlroot_respond.find("vm:Children/", namespaces)
+ if children_section is not None:
+ vCloud_extension_section = children_section.find(
+ "xmlns:VCloudExtension", namespaces
+ )
+ if vCloud_extension_section is not None:
+ vm_vcenter_info = {}
+ vim_info = vCloud_extension_section.find(
+ "vmext:VmVimInfo", namespaces
+ )
+ vmext = vim_info.find("vmext:VmVimObjectRef", namespaces)
+ if vmext is not None:
+ vm_vcenter_info["vm_moref_id"] = vmext.find(
+ "vmext:MoRef", namespaces
+ ).text
+ parsed_respond["vm_vcenter_info"] = vm_vcenter_info
+
+ except Exception as exp:
+ log.warning(
+ "Error occurred for getting vApp details: {}\n{}".format(
+ exp, traceback.format_exc()
+ )
+ )
+
+ return parsed_respond
+
+ def collect(self, vnfr: dict):
+ vnfd = self.common_db.get_vnfd(vnfr["vnfd-id"])
+ vdu_mappings = {}
+
+ # Populate extra tags for metrics
+ nsr_id = vnfr["nsr-id-ref"]
+ tags = {}
+ tags["ns_name"] = self.common_db.get_nsr(nsr_id)["name"]
+ if vnfr["_admin"]["projects_read"]:
+ tags["project_id"] = vnfr["_admin"]["projects_read"][0]
+ else:
+ tags["project_id"] = ""
+
+ # Fetch the list of all known resources from vROPS.
+ resource_list = self.vrops.get_vm_resource_list_from_vrops()
+
+ for vdur in vnfr["vdur"]:
+ # This avoids errors when vdur records have not been completely filled
+ if "name" not in vdur:
+ continue
+ vdu = next(filter(lambda vdu: vdu["id"] == vdur["vdu-id-ref"], vnfd["vdu"]))
+
+ if "monitoring-parameter" not in vdu:
+ continue
+
+ resource_uuid = vdur["vim-id"]
+ # Find vm_moref_id from vApp uuid in vCD
+ vim_id = self.get_vm_moref_id(resource_uuid)
+ if vim_id is None:
+ log.debug(
+ "Failed to find vROPS ID for vApp in vCD: {}".format(resource_uuid)
+ )
+ continue
+
+ vdu_mappings[vim_id] = {"name": vdur["name"]}
+
+ # Map the vROPS instance id to the vim-id so we can look it up.
+ for resource in resource_list:
+ for resourceIdentifier in resource["resourceKey"][
+ "resourceIdentifiers"
+ ]:
+ if (
+ resourceIdentifier["identifierType"]["name"]
+ == "VMEntityObjectID"
+ ):
+ if resourceIdentifier["value"] != vim_id:
+ continue
+ vdu_mappings[vim_id]["vrops_id"] = resource["identifier"]
+
+ if len(vdu_mappings) != 0:
+ return self.vrops.get_metrics(
+ vdu_mappings=vdu_mappings,
+ monitoring_params=vdu["monitoring-parameter"],
+ vnfr=vnfr,
+ tags=tags,
+ )
+ else:
+ return []
diff --git a/osm_mon/collector/vnf_collectors/vrops/__init__.py b/osm_mon/collector/vnf_collectors/vrops/__init__.py
new file mode 100644
index 0000000..30d864e
--- /dev/null
+++ b/osm_mon/collector/vnf_collectors/vrops/__init__.py
@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# #
+# Copyright 2016-2019 VMware Inc.
+# This file is part of ETSI OSM
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: osslegalrouting@vmware.com
+# #
diff --git a/osm_mon/collector/vnf_collectors/vrops/metrics.py b/osm_mon/collector/vnf_collectors/vrops/metrics.py
new file mode 100644
index 0000000..d4bd69f
--- /dev/null
+++ b/osm_mon/collector/vnf_collectors/vrops/metrics.py
@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+
+# #
+# Copyright 2016-2019 VMware Inc.
+# This file is part of ETSI OSM
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: osslegalrouting@vmware.com
+# #
+
+# Ref: https://docs.vmware.com/en/vRealize-Operations-Manager/7.0/vrealize-operations-manager-70-reference-guide.pdf
+# Potential metrics of interest
+# "cpu|capacity_contentionPct"
+# "cpu|corecount_provisioned"
+# "cpu|costopPct"
+# "cpu|demandmhz"
+# "cpu|demandPct"
+# "cpu|effective_limit"
+# "cpu|iowaitPct"
+# "cpu|readyPct"
+# "cpu|swapwaitPct"
+# "cpu|usage_average"
+# "cpu|usagemhz_average"
+# "cpu|usagemhz_average_mtd"
+# "cpu|vm_capacity_provisioned"
+# "cpu|workload"
+# "guestfilesystem|percentage_total"
+# "guestfilesystem|usage_total"
+# "mem|consumedPct"
+# "mem|guest_usage"
+# "mem|host_contentionPct"
+# "mem|reservation_used"
+# "mem|swapinRate_average"
+# "mem|swapoutRate_average"
+# "mem|swapped_average"
+# "mem|usage_average"
+# "net:Aggregate of all instances|droppedPct"
+# "net|broadcastTx_summation"
+# "net|droppedTx_summation"
+# "net|multicastTx_summation"
+# "net|pnicBytesRx_average"
+# "net|pnicBytesTx_average"
+# "net|received_average"
+# "net|transmitted_average"
+# "net|usage_average"
+# "virtualDisk:Aggregate of all instances|commandsAveraged_average"
+# "virtualDisk:Aggregate of all instances|numberReadAveraged_average"
+# "virtualDisk:Aggregate of all instances|numberWriteAveraged_average"
+# "virtualDisk:Aggregate of all instances|totalLatency"
+# "virtualDisk:Aggregate of all instances|totalReadLatency_average"
+# "virtualDisk:Aggregate of all instances|totalWriteLatency_average"
+# "virtualDisk:Aggregate of all instances|usage"
+# "virtualDisk:Aggregate of all instances|vDiskOIO"
+# "virtualDisk|read_average"
+# "virtualDisk|write_average"
+
+METRIC_MAPPINGS = {
+ # Percent guest operating system active memory.
+ "average_memory_utilization": "mem|usage_average",
+ # Percentage of CPU that was used out of all the CPU that was allocated.
+ "cpu_utilization": "cpu|usage_average",
+ # KB/s of data read in the performance interval
+ "disk_read_bytes": "virtualDisk|read_average",
+ # Average of read commands per second during the collection interval.
+ "disk_read_ops": "virtualDisk:aggregate of all instances|numberReadAveraged_average",
+ # KB/s of data written in the performance interval.
+ "disk_write_bytes": "virtualDisk|write_average",
+ # Average of write commands per second during the collection interval.
+ "disk_write_ops": "virtualDisk:aggregate of all instances|numberWriteAveraged_average",
+ # Not supported by vROPS, will always return 0.
+ "packets_in_dropped": "net|droppedRx_summation",
+ # Transmitted packets dropped in the collection interval.
+ "packets_out_dropped": "net|droppedTx_summation",
+ # Bytes received in the performance interval.
+ "packets_received": "net|received_average",
+ # Packets transmitted in the performance interval.
+ "packets_sent": "net|transmitted_average",
+}
diff --git a/osm_mon/collector/vnf_collectors/vrops/vrops_helper.py b/osm_mon/collector/vnf_collectors/vrops/vrops_helper.py
new file mode 100644
index 0000000..eadd5c7
--- /dev/null
+++ b/osm_mon/collector/vnf_collectors/vrops/vrops_helper.py
@@ -0,0 +1,214 @@
+# -*- coding: utf-8 -*-
+
+# #
+# Copyright 2016-2019 VMware Inc.
+# This file is part of ETSI OSM
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: osslegalrouting@vmware.com
+# #
+
+import logging
+import json
+import requests
+import traceback
+
+from osm_mon.collector.vnf_metric import VnfMetric
+from osm_mon.collector.vnf_collectors.vrops.metrics import METRIC_MAPPINGS
+import copy
+
+log = logging.getLogger(__name__)
+
+
+# If the unit from vROPS does not align with the expected value. multiply by the specified amount to ensure
+# the correct unit is returned.
+METRIC_MULTIPLIERS = {
+ "disk_read_bytes": 1024,
+ "disk_write_bytes": 1024,
+ "packets_received": 1024,
+ "packets_sent": 1024,
+}
+
+
+class vROPS_Helper:
+ def __init__(
+ self,
+ vrops_site="https://vrops",
+ vrops_user="",
+ vrops_password="",
+ verify_ssl=False,
+ ):
+ self.vrops_site = vrops_site
+ self.vrops_user = vrops_user
+ self.vrops_password = vrops_password
+ self.verify_ssl = verify_ssl
+
+ def get_vrops_token(self):
+ """Fetches token from vrops"""
+ auth_url = "/suite-api/api/auth/token/acquire"
+ headers = {"Content-Type": "application/json", "Accept": "application/json"}
+ req_body = {"username": self.vrops_user, "password": self.vrops_password}
+ resp = requests.post(
+ self.vrops_site + auth_url,
+ json=req_body,
+ verify=self.verify_ssl,
+ headers=headers,
+ )
+ if resp.status_code != 200:
+ log.error(
+ "Failed to get token from vROPS: {} {}".format(
+ resp.status_code, resp.content
+ )
+ )
+ return None
+
+ resp_data = json.loads(resp.content.decode("utf-8"))
+ return resp_data["token"]
+
+ def get_vm_resource_list_from_vrops(self):
+ """Find all known resource IDs in vROPs"""
+ auth_token = self.get_vrops_token()
+ api_url = "/suite-api/api/resources?resourceKind=VirtualMachine"
+ headers = {
+ "Accept": "application/json",
+ "Authorization": "vRealizeOpsToken {}".format(auth_token),
+ }
+ resource_list = []
+
+ resp = requests.get(
+ self.vrops_site + api_url, verify=self.verify_ssl, headers=headers
+ )
+
+ if resp.status_code != 200:
+ log.error(
+ "Failed to get resource list from vROPS: {} {}".format(
+ resp.status_code, resp.content
+ )
+ )
+ return resource_list
+
+ try:
+ resp_data = json.loads(resp.content.decode("utf-8"))
+ if resp_data.get("resourceList") is not None:
+ resource_list = resp_data.get("resourceList")
+
+ except Exception as exp:
+ log.error(
+ "get_vm_resource_id: Error in parsing {}\n{}".format(
+ exp, traceback.format_exc()
+ )
+ )
+
+ return resource_list
+
+ def get_metrics(self, vdu_mappings={}, monitoring_params={}, vnfr=None, tags={}):
+ monitoring_keys = {}
+ # Collect the names of all the metrics we need to query
+ for metric_entry in monitoring_params:
+ metric_name = metric_entry["performance-metric"]
+ if metric_name not in METRIC_MAPPINGS:
+ log.debug("Metric {} not supported, ignoring".format(metric_name))
+ continue
+ monitoring_keys[metric_name] = METRIC_MAPPINGS[metric_name]
+
+ metrics = []
+ # Make a query for only the stats we have been asked for
+ stats_key = ""
+ for stat in monitoring_keys.values():
+ stats_key += "&statKey={}".format(stat)
+
+ # And only ask for the resource ids that we are interested in
+ resource_ids = ""
+ sanitized_vdu_mappings = copy.deepcopy(vdu_mappings)
+ for key in vdu_mappings.keys():
+ vdu = vdu_mappings[key]
+ if "vrops_id" not in vdu:
+ log.info("Could not find vROPS id for vdu {}".format(vdu))
+ del sanitized_vdu_mappings[key]
+ continue
+ resource_ids += "&resourceId={}".format(vdu["vrops_id"])
+ vdu_mappings = sanitized_vdu_mappings
+
+ try:
+ # Now we can make a single call to vROPS to collect all relevant metrics for resources we need to monitor
+ api_url = (
+ "/suite-api/api/resources/stats?IntervalType=MINUTES&IntervalCount=1"
+ "&rollUpType=MAX¤tOnly=true{}{}".format(stats_key, resource_ids)
+ )
+
+ auth_token = self.get_vrops_token()
+ headers = {
+ "Accept": "application/json",
+ "Authorization": "vRealizeOpsToken {}".format(auth_token),
+ }
+
+ resp = requests.get(
+ self.vrops_site + api_url, verify=self.verify_ssl, headers=headers
+ )
+
+ if resp.status_code != 200:
+ log.error(
+ "Failed to get Metrics data from vROPS for {} {}".format(
+ resp.status_code, resp.content
+ )
+ )
+ return metrics
+
+ m_data = json.loads(resp.content.decode("utf-8"))
+ if "values" not in m_data:
+ return metrics
+
+ statistics = m_data["values"]
+ for vdu_stat in statistics:
+ vrops_id = vdu_stat["resourceId"]
+ vdu_name = None
+ for vdu in vdu_mappings.values():
+ if vdu["vrops_id"] == vrops_id:
+ vdu_name = vdu["name"]
+ if vdu_name is None:
+ continue
+ for item in vdu_stat["stat-list"]["stat"]:
+ reported_metric = item["statKey"]["key"]
+ if reported_metric not in METRIC_MAPPINGS.values():
+ continue
+
+ # Convert the vROPS metric name back to OSM key
+ metric_name = list(METRIC_MAPPINGS.keys())[
+ list(METRIC_MAPPINGS.values()).index(reported_metric)
+ ]
+ if metric_name in monitoring_keys.keys():
+ metric_value = item["data"][-1]
+ if metric_name in METRIC_MULTIPLIERS:
+ metric_value *= METRIC_MULTIPLIERS[metric_name]
+ metric = VnfMetric(
+ vnfr["nsr-id-ref"],
+ vnfr["member-vnf-index-ref"],
+ vdu_name,
+ metric_name,
+ metric_value,
+ tags,
+ )
+
+ metrics.append(metric)
+
+ except Exception as exp:
+ log.error(
+ "Exception while parsing metrics data from vROPS {}\n{}".format(
+ exp, traceback.format_exc()
+ )
+ )
+
+ return metrics
diff --git a/osm_mon/collector/vnf_metric.py b/osm_mon/collector/vnf_metric.py
new file mode 100644
index 0000000..961a4ef
--- /dev/null
+++ b/osm_mon/collector/vnf_metric.py
@@ -0,0 +1,40 @@
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+from osm_mon.collector.metric import Metric
+import logging
+
+log = logging.getLogger(__name__)
+
+
+class VnfMetric(Metric):
+ def __init__(
+ self, nsr_id, vnf_member_index, vdur_name, name, value, extra_tags: dict = None
+ ):
+ tags = {
+ "ns_id": nsr_id,
+ "vnf_member_index": vnf_member_index,
+ "vdu_name": vdur_name,
+ }
+ if extra_tags:
+ tags.update(extra_tags)
+ log.debug("Tags: %s", tags)
+ super().__init__(tags, name, value)