osm_mon/collector/service.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Copyright 2018 Whitestack, LLC
   4 # *************************************************************
   5
   6 # This file is part of OSM Monitoring module
   7 # All Rights Reserved to Whitestack, LLC
   8
   9 # Licensed under the Apache License, Version 2.0 (the "License"); you may
  10 # not use this file except in compliance with the License. You may obtain
  11 # a copy of the License at
  12
  13 #         http://www.apache.org/licenses/LICENSE-2.0
  14
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  17 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  18 # License for the specific language governing permissions and limitations
  19 # under the License.
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact: bdiaz@whitestack.com or glavado@whitestack.com
  22 ##
  23 import logging
  24 import multiprocessing
  25 from typing import List
  26
  27 from osm_mon.collector.infra_collectors.onos import OnosInfraCollector
  28 from osm_mon.collector.infra_collectors.openstack import OpenstackInfraCollector
  29 from osm_mon.collector.infra_collectors.vio import VIOInfraCollector
  30 from osm_mon.collector.infra_collectors.vmware import VMwareInfraCollector
  31 from osm_mon.collector.metric import Metric
  32 from osm_mon.collector.vnf_collectors.juju import VCACollector
  33 from osm_mon.collector.vnf_collectors.openstack import OpenstackCollector
  34 from osm_mon.collector.vnf_collectors.vio import VIOCollector
  35 from osm_mon.collector.vnf_collectors.vmware import VMwareCollector
  36 from osm_mon.core.common_db import CommonDbClient
  37 from osm_mon.core.config import Config
  38
  39 log = logging.getLogger(__name__)
  40
  41 VIM_COLLECTORS = {
  42     "openstack": OpenstackCollector,
  43     "vmware": VMwareCollector,
  44     "vio": VIOCollector
  45 }
  46 VIM_INFRA_COLLECTORS = {
  47     "openstack": OpenstackInfraCollector,
  48     "vmware": VMwareInfraCollector,
  49     "vio": VIOInfraCollector
  50 }
  51 SDN_INFRA_COLLECTORS = {
  52     "onosof": OnosInfraCollector,
  53     "onos_vpls": OnosInfraCollector
  54 }
  55
  56
  57 class CollectorService:
  58     def __init__(self, config: Config):
  59         self.conf = config
  60         self.common_db = CommonDbClient(self.conf)
  61         self.queue = multiprocessing.Queue()
  62
  63     def _collect_vim_metrics(self, vnfr: dict, vim_account_id: str):
  64         # TODO(diazb) Add support for aws
  65         vim_type = self._get_vim_type(vim_account_id)
  66         if vim_type in VIM_COLLECTORS:
  67             collector = VIM_COLLECTORS[vim_type](self.conf, vim_account_id)
  68             metrics = collector.collect(vnfr)
  69             for metric in metrics:
  70                 self.queue.put(metric)
  71         else:
  72             log.debug("vimtype %s is not supported.", vim_type)
  73
  74     def _collect_vim_infra_metrics(self, vim_account_id: str):
  75         vim_type = self._get_vim_type(vim_account_id)
  76         if vim_type in VIM_INFRA_COLLECTORS:
  77             collector = VIM_INFRA_COLLECTORS[vim_type](self.conf, vim_account_id)
  78             metrics = collector.collect()
  79             for metric in metrics:
  80                 self.queue.put(metric)
  81         else:
  82             log.debug("vimtype %s is not supported.", vim_type)
  83
  84     def _collect_sdnc_infra_metrics(self, sdnc_id: str):
  85         common_db = CommonDbClient(self.conf)
  86         sdn_type = common_db.get_sdnc(sdnc_id)['type']
  87         if sdn_type in SDN_INFRA_COLLECTORS:
  88             collector = SDN_INFRA_COLLECTORS[sdn_type](self.conf, sdnc_id)
  89             metrics = collector.collect()
  90             for metric in metrics:
  91                 self.queue.put(metric)
  92         else:
  93             log.debug("sdn_type %s is not supported.", sdn_type)
  94
  95     def _collect_vca_metrics(self, vnfr: dict):
  96         log.debug('_collect_vca_metrics')
  97         log.debug('vnfr: %s', vnfr)
  98         vca_collector = VCACollector(self.conf)
  99         metrics = vca_collector.collect(vnfr)
 100         for metric in metrics:
 101             self.queue.put(metric)
 102
 103     def collect_metrics(self) -> List[Metric]:
 104         vnfrs = self.common_db.get_vnfrs()
 105         processes = []
 106         for vnfr in vnfrs:
 107             nsr_id = vnfr['nsr-id-ref']
 108             vnf_member_index = vnfr['member-vnf-index-ref']
 109             vim_account_id = self.common_db.get_vim_account_id(nsr_id, vnf_member_index)
 110             p = multiprocessing.Process(target=self._collect_vim_metrics,
 111                                         args=(vnfr, vim_account_id))
 112             processes.append(p)
 113             p.start()
 114             p = multiprocessing.Process(target=self._collect_vca_metrics,
 115                                         args=(vnfr,))
 116             processes.append(p)
 117             p.start()
 118         vims = self.common_db.get_vim_accounts()
 119         for vim in vims:
 120             p = multiprocessing.Process(target=self._collect_vim_infra_metrics,
 121                                         args=(vim['_id'],))
 122             processes.append(p)
 123             p.start()
 124         sdncs = self.common_db.get_sdncs()
 125         for sdnc in sdncs:
 126             p = multiprocessing.Process(target=self._collect_sdnc_infra_metrics,
 127                                         args=(sdnc['_id'],))
 128             processes.append(p)
 129             p.start()
 130         for process in processes:
 131             process.join(timeout=20)
 132         for process in processes:
 133             if process.is_alive():
 134                 process.terminate()
 135         metrics = []
 136         while not self.queue.empty():
 137             metrics.append(self.queue.get())
 138         return metrics
 139
 140     def _get_vim_type(self, vim_account_id: str) -> str:
 141         common_db = CommonDbClient(self.conf)
 142         vim_account = common_db.get_vim_account(vim_account_id)
 143         vim_type = vim_account['vim_type']
 144         if 'config' in vim_account and 'vim_type' in vim_account['config']:
 145             vim_type = vim_account['config']['vim_type'].lower()
 146             if vim_type == 'vio' and 'vrops_site' not in vim_account['config']:
 147                 vim_type = 'openstack'
 148         return vim_type