1 # -*- coding: utf-8 -*-
3 # Copyright 2018 Whitestack, LLC
4 # *************************************************************
6 # This file is part of OSM Monitoring module
7 # All Rights Reserved to Whitestack, LLC
9 # Licensed under the Apache License, Version 2.0 (the "License"); you may
10 # not use this file except in compliance with the License. You may obtain
11 # a copy of the License at
13 # http://www.apache.org/licenses/LICENSE-2.0
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
17 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
18 # License for the specific language governing permissions and limitations
20 # For those usages not covered by the Apache License, Version 2.0 please
21 # contact: bdiaz@whitestack.com or glavado@whitestack.com
24 # This version uses a ProcessThreadPoolExecutor to limit the number of processes launched
27 from typing
import List
28 import concurrent
.futures
31 from osm_mon
.collector
.infra_collectors
.onos
import OnosInfraCollector
32 from osm_mon
.collector
.infra_collectors
.openstack
import OpenstackInfraCollector
33 from osm_mon
.collector
.infra_collectors
.vio
import VIOInfraCollector
34 from osm_mon
.collector
.infra_collectors
.vmware
import VMwareInfraCollector
35 from osm_mon
.collector
.metric
import Metric
36 from osm_mon
.collector
.vnf_collectors
.juju
import VCACollector
37 from osm_mon
.collector
.vnf_collectors
.openstack
import OpenstackCollector
38 from osm_mon
.collector
.vnf_collectors
.vio
import VIOCollector
39 from osm_mon
.collector
.vnf_collectors
.vmware
import VMwareCollector
40 from osm_mon
.core
.common_db
import CommonDbClient
41 from osm_mon
.core
.config
import Config
43 log
= logging
.getLogger(__name__
)
46 "openstack": OpenstackCollector
,
47 "vmware": VMwareCollector
,
50 VIM_INFRA_COLLECTORS
= {
51 "openstack": OpenstackInfraCollector
,
52 "vmware": VMwareInfraCollector
,
53 "vio": VIOInfraCollector
55 SDN_INFRA_COLLECTORS
= {
56 "onosof": OnosInfraCollector
,
57 "onos_vpls": OnosInfraCollector
61 class CollectorService
:
63 def __init__(self
, config
: Config
):
65 self
.common_db
= CommonDbClient(self
.conf
)
68 # static methods to be executed in the Processes
70 def _get_vim_type(conf
: Config
, vim_account_id
: str) -> str:
71 common_db
= CommonDbClient(conf
)
72 vim_account
= common_db
.get_vim_account(vim_account_id
)
73 vim_type
= vim_account
['vim_type']
74 if 'config' in vim_account
and 'vim_type' in vim_account
['config']:
75 vim_type
= vim_account
['config']['vim_type'].lower()
76 if vim_type
== 'vio' and 'vrops_site' not in vim_account
['config']:
77 vim_type
= 'openstack'
81 def _collect_vim_metrics(conf
: Config
, vnfr
: dict, vim_account_id
: str):
82 # TODO(diazb) Add support for aws
84 vim_type
= CollectorService
._get
_vim
_type
(conf
, vim_account_id
)
85 log
.debug("vim type.....{}".format(vim_type
))
86 if vim_type
in VIM_COLLECTORS
:
87 collector
= VIM_COLLECTORS
[vim_type
](conf
, vim_account_id
)
88 metrics
= collector
.collect(vnfr
)
89 log
.debug("Collecting vim metrics.....{}".format(metrics
))
91 log
.debug("vimtype %s is not supported.", vim_type
)
95 def _collect_vca_metrics(conf
: Config
, vnfr
: dict):
97 vca_collector
= VCACollector(conf
)
98 metrics
= vca_collector
.collect(vnfr
)
99 log
.debug("Collecting vca metrics.....{}".format(metrics
))
103 def _collect_vim_infra_metrics(conf
: Config
, vim_account_id
: str):
104 log
.info("Collecting vim infra metrics")
106 vim_type
= CollectorService
._get
_vim
_type
(conf
, vim_account_id
)
107 if vim_type
in VIM_INFRA_COLLECTORS
:
108 collector
= VIM_INFRA_COLLECTORS
[vim_type
](conf
, vim_account_id
)
109 metrics
= collector
.collect()
110 log
.debug("Collecting vim infra metrics.....{}".format(metrics
))
112 log
.debug("vimtype %s is not supported.", vim_type
)
116 def _collect_sdnc_infra_metrics(conf
: Config
, sdnc_id
: str):
117 log
.info("Collecting sdnc metrics")
119 common_db
= CommonDbClient(conf
)
120 sdn_type
= common_db
.get_sdnc(sdnc_id
)['type']
121 if sdn_type
in SDN_INFRA_COLLECTORS
:
122 collector
= SDN_INFRA_COLLECTORS
[sdn_type
](conf
, sdnc_id
)
123 metrics
= collector
.collect()
124 log
.debug("Collecting sdnc metrics.....{}".format(metrics
))
126 log
.debug("sdn_type %s is not supported.", sdn_type
)
130 def _stop_process_pool(executor
):
131 log
.info('Shutting down process pool')
133 log
.debug('Stopping residual processes in the process pool')
134 for pid
, process
in executor
._processes
.items():
135 if process
.is_alive():
137 except Exception as e
:
138 log
.info("Exception during process termination")
139 log
.debug("Exception %s" % (e
))
142 # Shutting down executor
143 log
.debug('Shutting down process pool executor')
145 except RuntimeError as e
:
146 log
.info('RuntimeError in shutting down executer')
147 log
.debug('RuntimeError %s' % (e
))
150 def collect_metrics(self
) -> List
[Metric
]:
151 vnfrs
= self
.common_db
.get_vnfrs()
154 start_time
= time
.time()
155 # Starting executor pool with pool size process_pool_size. Default process_pool_size is 20
156 with concurrent
.futures
.ProcessPoolExecutor(self
.conf
.get('collector', 'process_pool_size')) as executor
:
157 log
.info('Started metric collector process pool with pool size %s' % (self
.conf
.get('collector',
158 'process_pool_size')))
161 nsr_id
= vnfr
['nsr-id-ref']
162 vnf_member_index
= vnfr
['member-vnf-index-ref']
163 vim_account_id
= self
.common_db
.get_vim_account_id(nsr_id
, vnf_member_index
)
164 futures
.append(executor
.submit(CollectorService
._collect
_vim
_metrics
, self
.conf
, vnfr
, vim_account_id
))
165 futures
.append(executor
.submit(CollectorService
._collect
_vca
_metrics
, self
.conf
, vnfr
))
167 vims
= self
.common_db
.get_vim_accounts()
169 futures
.append(executor
.submit(CollectorService
._collect
_vim
_infra
_metrics
, self
.conf
, vim
['_id']))
171 sdncs
= self
.common_db
.get_sdncs()
173 futures
.append(executor
.submit(CollectorService
._collect
_sdnc
_infra
_metrics
, self
.conf
, sdnc
['_id']))
176 # Wait for future calls to complete till process_execution_timeout. Default is 50 seconds
177 for future
in concurrent
.futures
.as_completed(futures
, self
.conf
.get('collector',
178 'process_execution_timeout')):
179 result
= future
.result(timeout
=int(self
.conf
.get('collector',
180 'process_execution_timeout')))
181 metrics
.extend(result
)
182 log
.debug('result = %s' % (result
))
183 except concurrent
.futures
.TimeoutError
as e
:
184 # Some processes have not completed due to timeout error
185 log
.info('Some processes have not finished due to TimeoutError exception')
186 log
.debug('concurrent.futures.TimeoutError exception %s' % (e
))
188 # Shutting down process pool executor
189 CollectorService
._stop
_process
_pool
(executor
)
191 end_time
= time
.time()
192 log
.info("Collection completed in %s seconds", end_time
- start_time
)