Fix bug 1884 MON openstack token optimization

Reuse the openstack session created in infra collection,
thereby collecting infra and vnf metrics per vim account.

Change-Id: I7d9c9e32b366822101699f56cbf6fc8f8ec6f233
Signed-off-by: preethika.p <preethika.p@tataelxsi.co.in>

diff --git a/osm_mon/collector/infra_collectors/base_osinfra.py b/osm_mon/collector/infra_collectors/base_osinfra.py
index 7ea453d..6926447 100644 (file)
--- a/osm_mon/collector/infra_collectors/base_osinfra.py
+++ b/osm_mon/collector/infra_collectors/base_osinfra.py
@@ -42,7 +42,8 @@ class BaseOpenStackInfraCollector(BaseVimInfraCollector):
         self.conf = config
         self.common_db = CommonDbClient(config)
         self.vim_account = self.common_db.get_vim_account(vim_account_id)
-        self.keystone = self._build_keystone_client(self.vim_account)
+        # self.keystone = self._build_keystone_client(self.vim_account)
+        self.vim_session = None
         self.nova = self._build_nova_client(self.vim_account)
         self.cinder = self._build_cinder_client(self.vim_account)
         self.neutron, self.tenant_id = self._build_neutron_client(self.vim_account)
@@ -177,13 +178,14 @@ class BaseOpenStackInfraCollector(BaseVimInfraCollector):
 
     def _build_nova_client(self, vim_account: dict) -> nova_client.Client:
         sess = OpenstackUtils.get_session(vim_account)
+        self.vim_session = sess
         return nova_client.Client("2", session=sess, timeout=10)
 
     def _build_cinder_client(self, vim_account: dict) -> cinder_client.Client:
-        sess = OpenstackUtils.get_session(vim_account)
-        return cinder_client.Client("3", session=sess, timeout=10)
+        # sess = OpenstackUtils.get_session(vim_account)
+        return cinder_client.Client("3", session=self.vim_session, timeout=10)
 
     def _build_neutron_client(self, vim_account: dict) -> tuple:
-        sess = OpenstackUtils.get_session(vim_account)
-        tenant_id = sess.get_project_id()
-        return neutron_client.Client("2", session=sess, timeout=10), tenant_id
+        # sess = OpenstackUtils.get_session(vim_account)
+        tenant_id = self.vim_session.get_project_id()
+        return neutron_client.Client("2", session=self.vim_session, timeout=10), tenant_id

diff --git a/osm_mon/collector/service.py b/osm_mon/collector/service.py
index 132e9da..314ce11 100644 (file)
--- a/osm_mon/collector/service.py
+++ b/osm_mon/collector/service.py
@@ -55,6 +55,15 @@ VIM_INFRA_COLLECTORS = {
 }
 SDN_INFRA_COLLECTORS = {"onosof": OnosInfraCollector, "onos_vpls": OnosInfraCollector}
 
+# Map to store vim ids and corresponding vim session objects
+vim_sess_map = {}
+
+
+# Invoked from process executor to initialize the vim session map
+def init_session(session_map: dict):
+    global vim_sess_map
+    vim_sess_map = session_map
+
 
 class CollectorService:
     def __init__(self, config: Config):
@@ -81,7 +90,7 @@ class CollectorService:
         vim_type = CollectorService._get_vim_type(conf, vim_account_id)
         log.debug("vim type.....{}".format(vim_type))
         if vim_type in VIM_COLLECTORS:
-            collector = VIM_COLLECTORS[vim_type](conf, vim_account_id)
+            collector = VIM_COLLECTORS[vim_type](conf, vim_account_id, vim_sess_map[vim_account_id])
             metrics = collector.collect(vnfr)
             log.debug("Collecting vim metrics.....{}".format(metrics))
         else:
@@ -148,10 +157,26 @@ class CollectorService:
         vnfrs = self.common_db.get_vnfrs()
         metrics = []
 
+        # Get all vim ids regiestered in osm and create their corresponding vim session objects
+        # Vim ids and their corresponding session objects are stored in vim-session-map
+        # It optimizes the number of authentication tokens created in vim for metric colleciton
+        vim_sess_map.clear()
+        vims = self.common_db.get_vim_accounts()
+        for vim in vims:
+            vim_type = CollectorService._get_vim_type(self.conf, vim["_id"])
+            if vim_type in VIM_INFRA_COLLECTORS:
+                collector = VIM_INFRA_COLLECTORS[vim_type](self.conf, vim["_id"])
+                vim_sess = collector.vim_session if vim_type == "openstack" else None
+                # Populate the vim session map with vim ids and corresponding session objects
+                # vim session objects are stopred only for vim type openstack
+                if vim_sess:
+                    vim_sess_map[vim["_id"]] = vim_sess
+
         start_time = time.time()
         # Starting executor pool with pool size process_pool_size. Default process_pool_size is 20
+        # init_session is called to assign the session map to the gloabal vim session map variable
         with concurrent.futures.ProcessPoolExecutor(
-            self.conf.get("collector", "process_pool_size")
+            self.conf.get("collector", "process_pool_size"), initializer=init_session, initargs=(vim_sess_map,)
         ) as executor:
             log.info(
                 "Started metric collector process pool with pool size %s"
@@ -178,7 +203,6 @@ class CollectorService:
                     )
                 )
 
-            vims = self.common_db.get_vim_accounts()
             for vim in vims:
                 futures.append(
                     executor.submit(

diff --git a/osm_mon/collector/vnf_collectors/openstack.py b/osm_mon/collector/vnf_collectors/openstack.py
index 5119f33..ef366f9 100644 (file)
--- a/osm_mon/collector/vnf_collectors/openstack.py
+++ b/osm_mon/collector/vnf_collectors/openstack.py
@@ -84,11 +84,11 @@ class MetricType(Enum):
 
 
 class OpenstackCollector(BaseVimCollector):
-    def __init__(self, config: Config, vim_account_id: str):
+    def __init__(self, config: Config, vim_account_id: str, vim_session: object):
         super().__init__(config, vim_account_id)
         self.common_db = CommonDbClient(config)
         vim_account = self.common_db.get_vim_account(vim_account_id)
-        self.backend = self._get_backend(vim_account)
+        self.backend = self._get_backend(vim_account, vim_session)
 
     def _build_keystone_client(self, vim_account: dict) -> keystone_client.Client:
         sess = OpenstackUtils.get_session(vim_account)
@@ -182,14 +182,14 @@ class OpenstackCollector(BaseVimCollector):
                         log.info("Error in metric collection: %s" % e)
         return metrics
 
-    def _get_backend(self, vim_account: dict):
+    def _get_backend(self, vim_account: dict, vim_session: object):
         try:
-            gnocchi = GnocchiBackend(vim_account)
+            gnocchi = GnocchiBackend(vim_account, vim_session)
             gnocchi.client.metric.list(limit=1)
             log.info("Using gnocchi backend to collect metric")
             return gnocchi
         except (HTTPException, EndpointNotFound):
-            ceilometer = CeilometerBackend(vim_account)
+            ceilometer = CeilometerBackend(vim_account, vim_session)
             ceilometer.client.capabilities.get()
             log.info("Using ceilometer backend to collect metric")
             return ceilometer
@@ -209,17 +209,15 @@ class OpenstackBackend:
 
 
 class GnocchiBackend(OpenstackBackend):
-    def __init__(self, vim_account: dict):
-        self.client = self._build_gnocchi_client(vim_account)
-        self.neutron = self._build_neutron_client(vim_account)
+    def __init__(self, vim_account: dict, vim_session: object):
+        self.client = self._build_gnocchi_client(vim_account, vim_session)
+        self.neutron = self._build_neutron_client(vim_account, vim_session)
 
-    def _build_gnocchi_client(self, vim_account: dict) -> gnocchi_client.Client:
-        sess = OpenstackUtils.get_session(vim_account)
-        return gnocchi_client.Client(session=sess)
+    def _build_gnocchi_client(self, vim_account: dict, vim_session: object) -> gnocchi_client.Client:
+        return gnocchi_client.Client(session=vim_session)
 
-    def _build_neutron_client(self, vim_account: dict) -> neutron_client.Client:
-        sess = OpenstackUtils.get_session(vim_account)
-        return neutron_client.Client(session=sess)
+    def _build_neutron_client(self, vim_account: dict, vim_session: object) -> neutron_client.Client:
+        return neutron_client.Client(session=vim_session)
 
     def collect_metric(
         self, metric_type: MetricType, metric_name: str, resource_id: str
@@ -322,12 +320,11 @@ class GnocchiBackend(OpenstackBackend):
 
 
 class CeilometerBackend(OpenstackBackend):
-    def __init__(self, vim_account: dict):
-        self.client = self._build_ceilometer_client(vim_account)
+    def __init__(self, vim_account: dict, vim_session: object):
+        self.client = self._build_ceilometer_client(vim_account, vim_session)
 
-    def _build_ceilometer_client(self, vim_account: dict) -> ceilometer_client.Client:
-        sess = OpenstackUtils.get_session(vim_account)
-        return ceilometer_client.Client("2", session=sess)
+    def _build_ceilometer_client(self, vim_account: dict, vim_session: object) -> ceilometer_client.Client:
+        return ceilometer_client.Client("2", session=vim_session)
 
     def collect_metric(
         self, metric_type: MetricType, metric_name: str, resource_id: str

diff --git a/osm_mon/collector/vnf_collectors/vio.py b/osm_mon/collector/vnf_collectors/vio.py
index 4f7d55a..130b253 100644 (file)
--- a/osm_mon/collector/vnf_collectors/vio.py
+++ b/osm_mon/collector/vnf_collectors/vio.py
@@ -32,7 +32,7 @@ log = logging.getLogger(__name__)
 
 
 class VIOCollector(BaseVimCollector):
-    def __init__(self, config: Config, vim_account_id: str):
+    def __init__(self, config: Config, vim_account_id: str, vim_session: object):
         super().__init__(config, vim_account_id)
         self.common_db = CommonDbClient(config)
         cfg = self.get_vim_account(vim_account_id)

diff --git a/osm_mon/collector/vnf_collectors/vmware.py b/osm_mon/collector/vnf_collectors/vmware.py
index 93592b7..19065b3 100644 (file)
--- a/osm_mon/collector/vnf_collectors/vmware.py
+++ b/osm_mon/collector/vnf_collectors/vmware.py
@@ -40,7 +40,7 @@ API_VERSION = "27.0"
 
 
 class VMwareCollector(BaseVimCollector):
-    def __init__(self, config: Config, vim_account_id: str):
+    def __init__(self, config: Config, vim_account_id: str, vim_session: object):
         super().__init__(config, vim_account_id)
         self.common_db = CommonDbClient(config)
         vim_account = self.get_vim_account(vim_account_id)

diff --git a/osm_mon/tests/unit/collector/test_collector_service.py b/osm_mon/tests/unit/collector/test_collector_service.py
index 63209ed..d3cd395 100644 (file)
--- a/osm_mon/tests/unit/collector/test_collector_service.py
+++ b/osm_mon/tests/unit/collector/test_collector_service.py
@@ -27,6 +27,7 @@ from osm_mon.collector.vnf_collectors.openstack import OpenstackCollector
 from osm_mon.collector.vnf_collectors.vio import VIOCollector
 from osm_mon.core.common_db import CommonDbClient
 from osm_mon.core.config import Config
+from osm_mon.collector.service import init_session
 
 
 @mock.patch.object(CommonDbClient, "__init__", lambda *args, **kwargs: None)
@@ -34,6 +35,8 @@ class CollectorServiceTest(TestCase):
     def setUp(self):
         super().setUp()
         self.config = Config()
+        mock_vim_session = mock.MagicMock()
+        init_session(mock_vim_session)
 
     @mock.patch.object(OpenstackCollector, "__init__", lambda *args, **kwargs: None)
     @mock.patch.object(OpenstackCollector, "collect")

diff --git a/osm_mon/tests/unit/collector/vnf_collectors/test_openstack.py b/osm_mon/tests/unit/collector/vnf_collectors/test_openstack.py
index b42e0d4..991d7f9 100644 (file)
--- a/osm_mon/tests/unit/collector/vnf_collectors/test_openstack.py
+++ b/osm_mon/tests/unit/collector/vnf_collectors/test_openstack.py
@@ -45,6 +45,7 @@ class CollectorTest(TestCase):
     def test_collect_gnocchi_rate_instance(self, build_gnocchi_client, _):
         mock_gnocchi_client = mock.Mock()
         mock_gnocchi_client.metric = mock.Mock()
+        mock_vim_session = mock.Mock()
         mock_gnocchi_client.metric.get_measures.return_value = [
             (
                 datetime.datetime(
@@ -73,7 +74,7 @@ class CollectorTest(TestCase):
         ]
         build_gnocchi_client.return_value = mock_gnocchi_client
 
-        backend = GnocchiBackend({"_id": "test_uuid"})
+        backend = GnocchiBackend({"_id": "test_uuid"}, mock_vim_session)
         value = backend._collect_instance_metric("cpu", "test_resource_id")
         self.assertEqual(value, 1.0)
         mock_gnocchi_client.metric.get_measures.assert_called_once_with(
@@ -87,6 +88,7 @@ class CollectorTest(TestCase):
     @mock.patch.object(GnocchiBackend, "_build_gnocchi_client")
     def test_collect_gnocchi_non_rate_instance(self, build_gnocchi_client, _):
         mock_gnocchi_client = mock.Mock()
+        mock_vim_session = mock.Mock()
         mock_gnocchi_client.metric.get_measures.return_value = [
             (
                 datetime.datetime(
@@ -115,7 +117,7 @@ class CollectorTest(TestCase):
         ]
         build_gnocchi_client.return_value = mock_gnocchi_client
 
-        backend = GnocchiBackend({"_id": "test_uuid"})
+        backend = GnocchiBackend({"_id": "test_uuid"}, mock_vim_session)
         value = backend._collect_instance_metric("memory.usage", "test_resource_id")
         self.assertEqual(value, 128)
         mock_gnocchi_client.metric.get_measures.assert_called_once_with(
@@ -129,12 +131,13 @@ class CollectorTest(TestCase):
     @mock.patch.object(GnocchiBackend, "_build_gnocchi_client")
     def test_collect_gnocchi_no_metric(self, build_gnocchi_client, _):
         mock_gnocchi_client = mock.Mock()
+        mock_vim_session = mock.Mock()
         mock_gnocchi_client.metric.get_measures.side_effect = (
             gnocchiclient.exceptions.NotFound()
         )
         build_gnocchi_client.return_value = mock_gnocchi_client
 
-        backend = GnocchiBackend({"_id": "test_uuid"})
+        backend = GnocchiBackend({"_id": "test_uuid"}, mock_vim_session)
         value = backend._collect_instance_metric("memory.usage", "test_resource_id")
         self.assertIsNone(value)
         mock_gnocchi_client.metric.get_measures.assert_called_once_with(
@@ -150,6 +153,7 @@ class CollectorTest(TestCase):
         self, build_gnocchi_client, build_neutron_client
     ):
         mock_gnocchi_client = mock.Mock()
+        mock_vim_session = mock.Mock()
         mock_gnocchi_client.resource.search.return_value = [
             {"id": "test_id_1"},
             {"id": "test_id_2"},
@@ -183,7 +187,7 @@ class CollectorTest(TestCase):
 
         build_gnocchi_client.return_value = mock_gnocchi_client
 
-        backend = GnocchiBackend({"_id": "test_uuid"})
+        backend = GnocchiBackend({"_id": "test_uuid"}, mock_vim_session)
         value = backend._collect_interface_all_metric(
             "packets_received", "test_resource_id"
         )

diff --git a/osm_mon/tests/unit/collector/vnf_collectors/vmware/test_vcd_collector.py b/osm_mon/tests/unit/collector/vnf_collectors/vmware/test_vcd_collector.py
index 3cac999..3d92fd5 100644 (file)
--- a/osm_mon/tests/unit/collector/vnf_collectors/vmware/test_vcd_collector.py
+++ b/osm_mon/tests/unit/collector/vnf_collectors/vmware/test_vcd_collector.py
@@ -48,9 +48,10 @@ class CollectorTest(TestCase):
     @mock.patch("osm_mon.collector.vnf_collectors.vmware.CommonDbClient")
     def setUp(self, mock_db, mock_get_vim_account):
         super().setUp()
+        mock_vim_session = mock.Mock()
         mock_get_vim_account.return_value = VIM_ACCOUNT
         self.collector = VMwareCollector(
-            Config(), "9de6df67-b820-48c3-bcae-ee4838c5c5f4"
+            Config(), "9de6df67-b820-48c3-bcae-ee4838c5c5f4", mock_vim_session
         )
         self.mock_db = mock_db
         with open(
@@ -199,9 +200,10 @@ class VApp_Details_Test(TestCase):
     def setUp(self, mock_db, mock_get_vim_account):
         super().setUp()
         self.mock_db = mock_db
+        mock_vim_session = mock.Mock()
         mock_get_vim_account.return_value = VIM_ACCOUNT
         self.collector = VMwareCollector(
-            Config(), "9de6df67-b820-48c3-bcae-ee4838c5c5f4"
+            Config(), "9de6df67-b820-48c3-bcae-ee4838c5c5f4", mock_vim_session
         )
 
     def tearDown(self):
@@ -272,9 +274,10 @@ class Get_VM_Moref_Test(TestCase):
     def setUp(self, mock_db, mock_get_vim_account):
         super().setUp()
         self.mock_db = mock_db
+        mock_vim_session = mock.Mock()
         mock_get_vim_account.return_value = VIM_ACCOUNT
         self.collector = VMwareCollector(
-            Config(), "9de6df67-b820-48c3-bcae-ee4838c5c5f4"
+            Config(), "9de6df67-b820-48c3-bcae-ee4838c5c5f4", mock_vim_session
         )
 
     def tearDown(self):

diff --git a/osm_mon/tests/unit/collector/vnf_collectors/vmware/test_vio_collector.py b/osm_mon/tests/unit/collector/vnf_collectors/vmware/test_vio_collector.py
index f838c5a..7241f71 100644 (file)
--- a/osm_mon/tests/unit/collector/vnf_collectors/vmware/test_vio_collector.py
+++ b/osm_mon/tests/unit/collector/vnf_collectors/vmware/test_vio_collector.py
@@ -48,8 +48,9 @@ class CollectorTest(TestCase):
     def setUp(self, mock_db, mock_get_vim_account):
         super().setUp()
         self.mock_db = mock_db
+        mock_vim_session = mock.Mock()
         mock_get_vim_account.return_value = VIM_ACCOUNT
-        self.collector = VIOCollector(Config(), "9de6df67-b820-48c3-bcae-ee4838c5c5f4")
+        self.collector = VIOCollector(Config(), "9de6df67-b820-48c3-bcae-ee4838c5c5f4", mock_vim_session)
         with open(
             os.path.join(os.path.dirname(__file__), "osm_mocks", "VNFR.json"), "r"
         ) as f: