From 94948de24a6ff7d03cce2168be5a83c0ed97196d Mon Sep 17 00:00:00 2001 From: "preethika.p" Date: Wed, 23 Feb 2022 05:27:13 +0000 Subject: [PATCH] Fix bug 1884 MON openstack token optimization Reuse the openstack session created in infra collection, thereby collecting infra and vnf metrics per vim account. Change-Id: I7d9c9e32b366822101699f56cbf6fc8f8ec6f233 Signed-off-by: preethika.p --- .../infra_collectors/base_osinfra.py | 14 ++++---- osm_mon/collector/service.py | 30 ++++++++++++++-- osm_mon/collector/vnf_collectors/openstack.py | 35 +++++++++---------- osm_mon/collector/vnf_collectors/vio.py | 2 +- osm_mon/collector/vnf_collectors/vmware.py | 2 +- .../unit/collector/test_collector_service.py | 3 ++ .../vnf_collectors/test_openstack.py | 12 ++++--- .../vmware/test_vcd_collector.py | 9 +++-- .../vmware/test_vio_collector.py | 3 +- 9 files changed, 72 insertions(+), 38 deletions(-) diff --git a/osm_mon/collector/infra_collectors/base_osinfra.py b/osm_mon/collector/infra_collectors/base_osinfra.py index 7ea453d..6926447 100644 --- a/osm_mon/collector/infra_collectors/base_osinfra.py +++ b/osm_mon/collector/infra_collectors/base_osinfra.py @@ -42,7 +42,8 @@ class BaseOpenStackInfraCollector(BaseVimInfraCollector): self.conf = config self.common_db = CommonDbClient(config) self.vim_account = self.common_db.get_vim_account(vim_account_id) - self.keystone = self._build_keystone_client(self.vim_account) + # self.keystone = self._build_keystone_client(self.vim_account) + self.vim_session = None self.nova = self._build_nova_client(self.vim_account) self.cinder = self._build_cinder_client(self.vim_account) self.neutron, self.tenant_id = self._build_neutron_client(self.vim_account) @@ -177,13 +178,14 @@ class BaseOpenStackInfraCollector(BaseVimInfraCollector): def _build_nova_client(self, vim_account: dict) -> nova_client.Client: sess = OpenstackUtils.get_session(vim_account) + self.vim_session = sess return nova_client.Client("2", session=sess, timeout=10) def _build_cinder_client(self, vim_account: dict) -> cinder_client.Client: - sess = OpenstackUtils.get_session(vim_account) - return cinder_client.Client("3", session=sess, timeout=10) + # sess = OpenstackUtils.get_session(vim_account) + return cinder_client.Client("3", session=self.vim_session, timeout=10) def _build_neutron_client(self, vim_account: dict) -> tuple: - sess = OpenstackUtils.get_session(vim_account) - tenant_id = sess.get_project_id() - return neutron_client.Client("2", session=sess, timeout=10), tenant_id + # sess = OpenstackUtils.get_session(vim_account) + tenant_id = self.vim_session.get_project_id() + return neutron_client.Client("2", session=self.vim_session, timeout=10), tenant_id diff --git a/osm_mon/collector/service.py b/osm_mon/collector/service.py index 132e9da..314ce11 100644 --- a/osm_mon/collector/service.py +++ b/osm_mon/collector/service.py @@ -55,6 +55,15 @@ VIM_INFRA_COLLECTORS = { } SDN_INFRA_COLLECTORS = {"onosof": OnosInfraCollector, "onos_vpls": OnosInfraCollector} +# Map to store vim ids and corresponding vim session objects +vim_sess_map = {} + + +# Invoked from process executor to initialize the vim session map +def init_session(session_map: dict): + global vim_sess_map + vim_sess_map = session_map + class CollectorService: def __init__(self, config: Config): @@ -81,7 +90,7 @@ class CollectorService: vim_type = CollectorService._get_vim_type(conf, vim_account_id) log.debug("vim type.....{}".format(vim_type)) if vim_type in VIM_COLLECTORS: - collector = VIM_COLLECTORS[vim_type](conf, vim_account_id) + collector = VIM_COLLECTORS[vim_type](conf, vim_account_id, vim_sess_map[vim_account_id]) metrics = collector.collect(vnfr) log.debug("Collecting vim metrics.....{}".format(metrics)) else: @@ -148,10 +157,26 @@ class CollectorService: vnfrs = self.common_db.get_vnfrs() metrics = [] + # Get all vim ids regiestered in osm and create their corresponding vim session objects + # Vim ids and their corresponding session objects are stored in vim-session-map + # It optimizes the number of authentication tokens created in vim for metric colleciton + vim_sess_map.clear() + vims = self.common_db.get_vim_accounts() + for vim in vims: + vim_type = CollectorService._get_vim_type(self.conf, vim["_id"]) + if vim_type in VIM_INFRA_COLLECTORS: + collector = VIM_INFRA_COLLECTORS[vim_type](self.conf, vim["_id"]) + vim_sess = collector.vim_session if vim_type == "openstack" else None + # Populate the vim session map with vim ids and corresponding session objects + # vim session objects are stopred only for vim type openstack + if vim_sess: + vim_sess_map[vim["_id"]] = vim_sess + start_time = time.time() # Starting executor pool with pool size process_pool_size. Default process_pool_size is 20 + # init_session is called to assign the session map to the gloabal vim session map variable with concurrent.futures.ProcessPoolExecutor( - self.conf.get("collector", "process_pool_size") + self.conf.get("collector", "process_pool_size"), initializer=init_session, initargs=(vim_sess_map,) ) as executor: log.info( "Started metric collector process pool with pool size %s" @@ -178,7 +203,6 @@ class CollectorService: ) ) - vims = self.common_db.get_vim_accounts() for vim in vims: futures.append( executor.submit( diff --git a/osm_mon/collector/vnf_collectors/openstack.py b/osm_mon/collector/vnf_collectors/openstack.py index 5119f33..ef366f9 100644 --- a/osm_mon/collector/vnf_collectors/openstack.py +++ b/osm_mon/collector/vnf_collectors/openstack.py @@ -84,11 +84,11 @@ class MetricType(Enum): class OpenstackCollector(BaseVimCollector): - def __init__(self, config: Config, vim_account_id: str): + def __init__(self, config: Config, vim_account_id: str, vim_session: object): super().__init__(config, vim_account_id) self.common_db = CommonDbClient(config) vim_account = self.common_db.get_vim_account(vim_account_id) - self.backend = self._get_backend(vim_account) + self.backend = self._get_backend(vim_account, vim_session) def _build_keystone_client(self, vim_account: dict) -> keystone_client.Client: sess = OpenstackUtils.get_session(vim_account) @@ -182,14 +182,14 @@ class OpenstackCollector(BaseVimCollector): log.info("Error in metric collection: %s" % e) return metrics - def _get_backend(self, vim_account: dict): + def _get_backend(self, vim_account: dict, vim_session: object): try: - gnocchi = GnocchiBackend(vim_account) + gnocchi = GnocchiBackend(vim_account, vim_session) gnocchi.client.metric.list(limit=1) log.info("Using gnocchi backend to collect metric") return gnocchi except (HTTPException, EndpointNotFound): - ceilometer = CeilometerBackend(vim_account) + ceilometer = CeilometerBackend(vim_account, vim_session) ceilometer.client.capabilities.get() log.info("Using ceilometer backend to collect metric") return ceilometer @@ -209,17 +209,15 @@ class OpenstackBackend: class GnocchiBackend(OpenstackBackend): - def __init__(self, vim_account: dict): - self.client = self._build_gnocchi_client(vim_account) - self.neutron = self._build_neutron_client(vim_account) + def __init__(self, vim_account: dict, vim_session: object): + self.client = self._build_gnocchi_client(vim_account, vim_session) + self.neutron = self._build_neutron_client(vim_account, vim_session) - def _build_gnocchi_client(self, vim_account: dict) -> gnocchi_client.Client: - sess = OpenstackUtils.get_session(vim_account) - return gnocchi_client.Client(session=sess) + def _build_gnocchi_client(self, vim_account: dict, vim_session: object) -> gnocchi_client.Client: + return gnocchi_client.Client(session=vim_session) - def _build_neutron_client(self, vim_account: dict) -> neutron_client.Client: - sess = OpenstackUtils.get_session(vim_account) - return neutron_client.Client(session=sess) + def _build_neutron_client(self, vim_account: dict, vim_session: object) -> neutron_client.Client: + return neutron_client.Client(session=vim_session) def collect_metric( self, metric_type: MetricType, metric_name: str, resource_id: str @@ -322,12 +320,11 @@ class GnocchiBackend(OpenstackBackend): class CeilometerBackend(OpenstackBackend): - def __init__(self, vim_account: dict): - self.client = self._build_ceilometer_client(vim_account) + def __init__(self, vim_account: dict, vim_session: object): + self.client = self._build_ceilometer_client(vim_account, vim_session) - def _build_ceilometer_client(self, vim_account: dict) -> ceilometer_client.Client: - sess = OpenstackUtils.get_session(vim_account) - return ceilometer_client.Client("2", session=sess) + def _build_ceilometer_client(self, vim_account: dict, vim_session: object) -> ceilometer_client.Client: + return ceilometer_client.Client("2", session=vim_session) def collect_metric( self, metric_type: MetricType, metric_name: str, resource_id: str diff --git a/osm_mon/collector/vnf_collectors/vio.py b/osm_mon/collector/vnf_collectors/vio.py index 4f7d55a..130b253 100644 --- a/osm_mon/collector/vnf_collectors/vio.py +++ b/osm_mon/collector/vnf_collectors/vio.py @@ -32,7 +32,7 @@ log = logging.getLogger(__name__) class VIOCollector(BaseVimCollector): - def __init__(self, config: Config, vim_account_id: str): + def __init__(self, config: Config, vim_account_id: str, vim_session: object): super().__init__(config, vim_account_id) self.common_db = CommonDbClient(config) cfg = self.get_vim_account(vim_account_id) diff --git a/osm_mon/collector/vnf_collectors/vmware.py b/osm_mon/collector/vnf_collectors/vmware.py index 93592b7..19065b3 100644 --- a/osm_mon/collector/vnf_collectors/vmware.py +++ b/osm_mon/collector/vnf_collectors/vmware.py @@ -40,7 +40,7 @@ API_VERSION = "27.0" class VMwareCollector(BaseVimCollector): - def __init__(self, config: Config, vim_account_id: str): + def __init__(self, config: Config, vim_account_id: str, vim_session: object): super().__init__(config, vim_account_id) self.common_db = CommonDbClient(config) vim_account = self.get_vim_account(vim_account_id) diff --git a/osm_mon/tests/unit/collector/test_collector_service.py b/osm_mon/tests/unit/collector/test_collector_service.py index 63209ed..d3cd395 100644 --- a/osm_mon/tests/unit/collector/test_collector_service.py +++ b/osm_mon/tests/unit/collector/test_collector_service.py @@ -27,6 +27,7 @@ from osm_mon.collector.vnf_collectors.openstack import OpenstackCollector from osm_mon.collector.vnf_collectors.vio import VIOCollector from osm_mon.core.common_db import CommonDbClient from osm_mon.core.config import Config +from osm_mon.collector.service import init_session @mock.patch.object(CommonDbClient, "__init__", lambda *args, **kwargs: None) @@ -34,6 +35,8 @@ class CollectorServiceTest(TestCase): def setUp(self): super().setUp() self.config = Config() + mock_vim_session = mock.MagicMock() + init_session(mock_vim_session) @mock.patch.object(OpenstackCollector, "__init__", lambda *args, **kwargs: None) @mock.patch.object(OpenstackCollector, "collect") diff --git a/osm_mon/tests/unit/collector/vnf_collectors/test_openstack.py b/osm_mon/tests/unit/collector/vnf_collectors/test_openstack.py index b42e0d4..991d7f9 100644 --- a/osm_mon/tests/unit/collector/vnf_collectors/test_openstack.py +++ b/osm_mon/tests/unit/collector/vnf_collectors/test_openstack.py @@ -45,6 +45,7 @@ class CollectorTest(TestCase): def test_collect_gnocchi_rate_instance(self, build_gnocchi_client, _): mock_gnocchi_client = mock.Mock() mock_gnocchi_client.metric = mock.Mock() + mock_vim_session = mock.Mock() mock_gnocchi_client.metric.get_measures.return_value = [ ( datetime.datetime( @@ -73,7 +74,7 @@ class CollectorTest(TestCase): ] build_gnocchi_client.return_value = mock_gnocchi_client - backend = GnocchiBackend({"_id": "test_uuid"}) + backend = GnocchiBackend({"_id": "test_uuid"}, mock_vim_session) value = backend._collect_instance_metric("cpu", "test_resource_id") self.assertEqual(value, 1.0) mock_gnocchi_client.metric.get_measures.assert_called_once_with( @@ -87,6 +88,7 @@ class CollectorTest(TestCase): @mock.patch.object(GnocchiBackend, "_build_gnocchi_client") def test_collect_gnocchi_non_rate_instance(self, build_gnocchi_client, _): mock_gnocchi_client = mock.Mock() + mock_vim_session = mock.Mock() mock_gnocchi_client.metric.get_measures.return_value = [ ( datetime.datetime( @@ -115,7 +117,7 @@ class CollectorTest(TestCase): ] build_gnocchi_client.return_value = mock_gnocchi_client - backend = GnocchiBackend({"_id": "test_uuid"}) + backend = GnocchiBackend({"_id": "test_uuid"}, mock_vim_session) value = backend._collect_instance_metric("memory.usage", "test_resource_id") self.assertEqual(value, 128) mock_gnocchi_client.metric.get_measures.assert_called_once_with( @@ -129,12 +131,13 @@ class CollectorTest(TestCase): @mock.patch.object(GnocchiBackend, "_build_gnocchi_client") def test_collect_gnocchi_no_metric(self, build_gnocchi_client, _): mock_gnocchi_client = mock.Mock() + mock_vim_session = mock.Mock() mock_gnocchi_client.metric.get_measures.side_effect = ( gnocchiclient.exceptions.NotFound() ) build_gnocchi_client.return_value = mock_gnocchi_client - backend = GnocchiBackend({"_id": "test_uuid"}) + backend = GnocchiBackend({"_id": "test_uuid"}, mock_vim_session) value = backend._collect_instance_metric("memory.usage", "test_resource_id") self.assertIsNone(value) mock_gnocchi_client.metric.get_measures.assert_called_once_with( @@ -150,6 +153,7 @@ class CollectorTest(TestCase): self, build_gnocchi_client, build_neutron_client ): mock_gnocchi_client = mock.Mock() + mock_vim_session = mock.Mock() mock_gnocchi_client.resource.search.return_value = [ {"id": "test_id_1"}, {"id": "test_id_2"}, @@ -183,7 +187,7 @@ class CollectorTest(TestCase): build_gnocchi_client.return_value = mock_gnocchi_client - backend = GnocchiBackend({"_id": "test_uuid"}) + backend = GnocchiBackend({"_id": "test_uuid"}, mock_vim_session) value = backend._collect_interface_all_metric( "packets_received", "test_resource_id" ) diff --git a/osm_mon/tests/unit/collector/vnf_collectors/vmware/test_vcd_collector.py b/osm_mon/tests/unit/collector/vnf_collectors/vmware/test_vcd_collector.py index 3cac999..3d92fd5 100644 --- a/osm_mon/tests/unit/collector/vnf_collectors/vmware/test_vcd_collector.py +++ b/osm_mon/tests/unit/collector/vnf_collectors/vmware/test_vcd_collector.py @@ -48,9 +48,10 @@ class CollectorTest(TestCase): @mock.patch("osm_mon.collector.vnf_collectors.vmware.CommonDbClient") def setUp(self, mock_db, mock_get_vim_account): super().setUp() + mock_vim_session = mock.Mock() mock_get_vim_account.return_value = VIM_ACCOUNT self.collector = VMwareCollector( - Config(), "9de6df67-b820-48c3-bcae-ee4838c5c5f4" + Config(), "9de6df67-b820-48c3-bcae-ee4838c5c5f4", mock_vim_session ) self.mock_db = mock_db with open( @@ -199,9 +200,10 @@ class VApp_Details_Test(TestCase): def setUp(self, mock_db, mock_get_vim_account): super().setUp() self.mock_db = mock_db + mock_vim_session = mock.Mock() mock_get_vim_account.return_value = VIM_ACCOUNT self.collector = VMwareCollector( - Config(), "9de6df67-b820-48c3-bcae-ee4838c5c5f4" + Config(), "9de6df67-b820-48c3-bcae-ee4838c5c5f4", mock_vim_session ) def tearDown(self): @@ -272,9 +274,10 @@ class Get_VM_Moref_Test(TestCase): def setUp(self, mock_db, mock_get_vim_account): super().setUp() self.mock_db = mock_db + mock_vim_session = mock.Mock() mock_get_vim_account.return_value = VIM_ACCOUNT self.collector = VMwareCollector( - Config(), "9de6df67-b820-48c3-bcae-ee4838c5c5f4" + Config(), "9de6df67-b820-48c3-bcae-ee4838c5c5f4", mock_vim_session ) def tearDown(self): diff --git a/osm_mon/tests/unit/collector/vnf_collectors/vmware/test_vio_collector.py b/osm_mon/tests/unit/collector/vnf_collectors/vmware/test_vio_collector.py index f838c5a..7241f71 100644 --- a/osm_mon/tests/unit/collector/vnf_collectors/vmware/test_vio_collector.py +++ b/osm_mon/tests/unit/collector/vnf_collectors/vmware/test_vio_collector.py @@ -48,8 +48,9 @@ class CollectorTest(TestCase): def setUp(self, mock_db, mock_get_vim_account): super().setUp() self.mock_db = mock_db + mock_vim_session = mock.Mock() mock_get_vim_account.return_value = VIM_ACCOUNT - self.collector = VIOCollector(Config(), "9de6df67-b820-48c3-bcae-ee4838c5c5f4") + self.collector = VIOCollector(Config(), "9de6df67-b820-48c3-bcae-ee4838c5c5f4", mock_vim_session) with open( os.path.join(os.path.dirname(__file__), "osm_mocks", "VNFR.json"), "r" ) as f: -- 2.17.1