src/emuvim/dcemulator/monitoring.py

   1 """
   2 Copyright (c) 2015 SONATA-NFV
   3 ALL RIGHTS RESERVED.
   4
   5 Licensed under the Apache License, Version 2.0 (the "License");
   6 you may not use this file except in compliance with the License.
   7 You may obtain a copy of the License at
   8
   9     http://www.apache.org/licenses/LICENSE-2.0
  10
  11 Unless required by applicable law or agreed to in writing, software
  12 distributed under the License is distributed on an "AS IS" BASIS,
  13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 See the License for the specific language governing permissions and
  15 limitations under the License.
  16
  17 Neither the name of the SONATA-NFV [, ANY ADDITIONAL AFFILIATION]
  18 nor the names of its contributors may be used to endorse or promote
  19 products derived from this software without specific prior written
  20 permission.
  21
  22 This work has been performed in the framework of the SONATA project,
  23 funded by the European Commission under Grant number 671517 through
  24 the Horizon 2020 and 5G-PPP programmes. The authors would like to
  25 acknowledge the contributions of their colleagues of the SONATA
  26 partner consortium (www.sonata-nfv.eu).
  27 """
  28
  29 import logging
  30 import sys
  31 from mininet.node import  OVSSwitch
  32 import ast
  33 import time
  34 from prometheus_client import start_http_server, Summary, Histogram, Gauge, Counter, REGISTRY, CollectorRegistry, \
  35     pushadd_to_gateway, push_to_gateway, delete_from_gateway
  36 import threading
  37 from subprocess import Popen, check_call
  38 import os
  39 import docker
  40 import json
  41
  42 logging.basicConfig(level=logging.INFO)
  43
  44 """
  45 class to read openflow stats from the Ryu controller of the DCNetwork
  46 """
  47
  48 PUSHGATEWAY_PORT = 9091
  49 # we cannot use port 8080 because ryu-ofrest api  is already using that one
  50 CADVISOR_PORT = 8081
  51
  52 COOKIE_MASK = 0xffffffff
  53
  54 class DCNetworkMonitor():
  55     def __init__(self, net):
  56         self.net = net
  57         self.dockercli = docker.from_env()
  58
  59         # pushgateway address
  60         self.pushgateway = 'localhost:{0}'.format(PUSHGATEWAY_PORT)
  61
  62         # supported Prometheus metrics
  63         self.registry = CollectorRegistry()
  64         self.prom_tx_packet_count = Gauge('sonemu_tx_count_packets', 'Total number of packets sent',
  65                                           ['vnf_name', 'vnf_interface', 'flow_id'], registry=self.registry)
  66         self.prom_rx_packet_count = Gauge('sonemu_rx_count_packets', 'Total number of packets received',
  67                                           ['vnf_name', 'vnf_interface', 'flow_id'], registry=self.registry)
  68         self.prom_tx_byte_count = Gauge('sonemu_tx_count_bytes', 'Total number of bytes sent',
  69                                         ['vnf_name', 'vnf_interface', 'flow_id'], registry=self.registry)
  70         self.prom_rx_byte_count = Gauge('sonemu_rx_count_bytes', 'Total number of bytes received',
  71                                         ['vnf_name', 'vnf_interface', 'flow_id'], registry=self.registry)
  72
  73         self.prom_metrics={'tx_packets':self.prom_tx_packet_count, 'rx_packets':self.prom_rx_packet_count,
  74                            'tx_bytes':self.prom_tx_byte_count,'rx_bytes':self.prom_rx_byte_count}
  75
  76         # list of installed metrics to monitor
  77         # each entry can contain this data
  78         '''
  79         {
  80         switch_dpid = 0
  81         vnf_name = None
  82         vnf_interface = None
  83         previous_measurement = 0
  84         previous_monitor_time = 0
  85         metric_key = None
  86         mon_port = None
  87         }
  88         '''
  89         self.monitor_lock = threading.Lock()
  90         self.monitor_flow_lock = threading.Lock()
  91         self.network_metrics = []
  92         self.flow_metrics = []
  93         self.skewmon_metrics = {}
  94
  95         # start monitoring thread
  96         self.start_monitoring = True
  97         self.monitor_thread = threading.Thread(target=self.get_network_metrics)
  98         self.monitor_thread.start()
  99
 100         self.monitor_flow_thread = threading.Thread(target=self.get_flow_metrics)
 101         self.monitor_flow_thread.start()
 102
 103         # helper tools
 104         # cAdvisor, Prometheus pushgateway are started as external container, to gather monitoring metric in son-emu
 105         self.pushgateway_process = self.start_PushGateway()
 106         self.cadvisor_process = self.start_cAdvisor()
 107
 108
 109     # first set some parameters, before measurement can start
 110     def setup_flow(self, vnf_name, vnf_interface=None, metric='tx_packets', cookie=0):
 111
 112         flow_metric = {}
 113
 114         # check if port is specified (vnf:port)
 115         if vnf_interface is None:
 116             # take first interface by default
 117             connected_sw = self.net.DCNetwork_graph.neighbors(vnf_name)[0]
 118             link_dict = self.net.DCNetwork_graph[vnf_name][connected_sw]
 119             vnf_interface = link_dict[0]['src_port_id']
 120
 121         flow_metric['vnf_name'] = vnf_name
 122         flow_metric['vnf_interface'] = vnf_interface
 123
 124         vnf_switch = None
 125         for connected_sw in self.net.DCNetwork_graph.neighbors(vnf_name):
 126             link_dict = self.net.DCNetwork_graph[vnf_name][connected_sw]
 127             for link in link_dict:
 128                 if link_dict[link]['src_port_id'] == vnf_interface:
 129                     # found the right link and connected switch
 130                     vnf_switch = connected_sw
 131                     flow_metric['mon_port'] = link_dict[link]['dst_port_nr']
 132                     break
 133
 134         if not vnf_switch:
 135             logging.exception("vnf switch of {0}:{1} not found!".format(vnf_name, vnf_interface))
 136             return "vnf switch of {0}:{1} not found!".format(vnf_name, vnf_interface)
 137
 138         try:
 139             # default port direction to monitor
 140             if metric is None:
 141                 metric = 'tx_packets'
 142
 143             next_node = self.net.getNodeByName(vnf_switch)
 144
 145             if not isinstance(next_node, OVSSwitch):
 146                 logging.info("vnf: {0} is not connected to switch".format(vnf_name))
 147                 return
 148
 149             flow_metric['previous_measurement'] = 0
 150             flow_metric['previous_monitor_time'] = 0
 151
 152             flow_metric['switch_dpid'] = int(str(next_node.dpid), 16)
 153             flow_metric['metric_key'] = metric
 154             flow_metric['cookie'] = cookie
 155
 156             self.monitor_flow_lock.acquire()
 157             self.flow_metrics.append(flow_metric)
 158             self.monitor_flow_lock.release()
 159
 160             logging.info('Started monitoring flow:{3} {2} on {0}:{1}'.format(vnf_name, vnf_interface, metric, cookie))
 161             return 'Started monitoring flow:{3} {2} on {0}:{1}'.format(vnf_name, vnf_interface, metric, cookie)
 162
 163         except Exception as ex:
 164             logging.exception("setup_metric error.")
 165             return ex.message
 166
 167     def stop_flow(self, vnf_name, vnf_interface=None, metric=None, cookie=0,):
 168
 169         # check if port is specified (vnf:port)
 170         if vnf_interface is None and metric is not None:
 171             # take first interface by default
 172             connected_sw = self.net.DCNetwork_graph.neighbors(vnf_name)[0]
 173             link_dict = self.net.DCNetwork_graph[vnf_name][connected_sw]
 174             vnf_interface = link_dict[0]['src_port_id']
 175
 176         for flow_dict in self.flow_metrics:
 177             if flow_dict['vnf_name'] == vnf_name and flow_dict['vnf_interface'] == vnf_interface \
 178                     and flow_dict['metric_key'] == metric and flow_dict['cookie'] == cookie:
 179
 180                 self.monitor_flow_lock.acquire()
 181
 182                 self.flow_metrics.remove(flow_dict)
 183
 184                 # set metric to NaN
 185                 self.prom_metrics[flow_dict['metric_key']]. \
 186                     labels(vnf_name=vnf_name, vnf_interface=vnf_interface, flow_id=cookie). \
 187                     set(float('nan'))
 188
 189                 delete_from_gateway(self.pushgateway, job='sonemu-SDNcontroller')
 190
 191                 self.monitor_flow_lock.release()
 192
 193                 logging.info('Stopped monitoring flow {3}: {2} on {0}:{1}'.format(vnf_name, vnf_interface, metric, cookie))
 194                 return 'Stopped monitoring flow {3}: {2} on {0}:{1}'.format(vnf_name, vnf_interface, metric, cookie)
 195
 196         return 'Error stopping monitoring flow: {0} on {1}:{2}'.format(metric, vnf_name, vnf_interface)
 197
 198
 199     # first set some parameters, before measurement can start
 200     def setup_metric(self, vnf_name, vnf_interface=None, metric='tx_packets'):
 201
 202         network_metric = {}
 203
 204         # check if port is specified (vnf:port)
 205         if vnf_interface is None or vnf_interface == '':
 206             # take first interface by default
 207             connected_sw = self.net.DCNetwork_graph.neighbors(vnf_name)[0]
 208             link_dict = self.net.DCNetwork_graph[vnf_name][connected_sw]
 209             vnf_interface = link_dict[0]['src_port_id']
 210
 211         network_metric['vnf_name'] = vnf_name
 212         network_metric['vnf_interface'] = vnf_interface
 213
 214         for connected_sw in self.net.DCNetwork_graph.neighbors(vnf_name):
 215             link_dict = self.net.DCNetwork_graph[vnf_name][connected_sw]
 216             for link in link_dict:
 217                 if link_dict[link]['src_port_id'] == vnf_interface:
 218                     # found the right link and connected switch
 219                     network_metric['mon_port'] = link_dict[link]['dst_port_nr']
 220                     break
 221
 222         if 'mon_port' not in network_metric:
 223             logging.exception("vnf interface {0}:{1} not found!".format(vnf_name,vnf_interface))
 224             return "vnf interface {0}:{1} not found!".format(vnf_name,vnf_interface)
 225
 226         try:
 227             # default port direction to monitor
 228             if metric is None:
 229                 metric = 'tx_packets'
 230
 231             vnf_switch = self.net.DCNetwork_graph.neighbors(str(vnf_name))
 232
 233             if len(vnf_switch) > 1:
 234                 logging.info("vnf: {0} has multiple ports".format(vnf_name))
 235                 return
 236             elif len(vnf_switch) == 0:
 237                 logging.info("vnf: {0} is not connected".format(vnf_name))
 238                 return
 239             else:
 240                 vnf_switch = vnf_switch[0]
 241             next_node = self.net.getNodeByName(vnf_switch)
 242
 243             if not isinstance(next_node, OVSSwitch):
 244                 logging.info("vnf: {0} is not connected to switch".format(vnf_name))
 245                 return
 246
 247             network_metric['previous_measurement'] = 0
 248             network_metric['previous_monitor_time'] = 0
 249
 250
 251             network_metric['switch_dpid'] = int(str(next_node.dpid), 16)
 252             network_metric['metric_key'] = metric
 253
 254             self.monitor_lock.acquire()
 255             self.network_metrics.append(network_metric)
 256             self.monitor_lock.release()
 257
 258
 259             logging.info('Started monitoring: {2} on {0}:{1}'.format(vnf_name, vnf_interface, metric))
 260             return 'Started monitoring: {2} on {0}:{1}'.format(vnf_name, vnf_interface, metric)
 261
 262         except Exception as ex:
 263             logging.exception("setup_metric error.")
 264             return ex.message
 265
 266     def stop_metric(self, vnf_name, vnf_interface=None, metric=None):
 267
 268         # check if port is specified (vnf:port)
 269         if vnf_interface is None and metric is not None:
 270             # take first interface by default
 271             connected_sw = self.net.DCNetwork_graph.neighbors(vnf_name)[0]
 272             link_dict = self.net.DCNetwork_graph[vnf_name][connected_sw]
 273             vnf_interface = link_dict[0]['src_port_id']
 274
 275         for metric_dict in self.network_metrics:
 276             if metric_dict['vnf_name'] == vnf_name and metric_dict['vnf_interface'] == vnf_interface \
 277                     and metric_dict['metric_key'] == metric:
 278
 279                 self.monitor_lock.acquire()
 280
 281                 self.network_metrics.remove(metric_dict)
 282
 283                 # set values to NaN, prometheus api currently does not support removal of metrics
 284                 #self.prom_metrics[metric_dict['metric_key']].labels(vnf_name, vnf_interface).set(float('nan'))
 285                 self.prom_metrics[metric_dict['metric_key']]. \
 286                     labels(vnf_name=vnf_name, vnf_interface=vnf_interface, flow_id=None). \
 287                     set(float('nan'))
 288
 289                 # this removes the complete metric, all labels...
 290                 # 1 single monitor job for all metrics of the SDN controller
 291                 # we can only  remove from the pushgateway grouping keys(labels) which we have defined for the add_to_pushgateway
 292                 # we can not specify labels from the metrics to be removed
 293                 # if we need to remove the metrics seperatelty, we need to give them a separate grouping key, and probably a diffferent registry also
 294                 delete_from_gateway(self.pushgateway, job='sonemu-SDNcontroller')
 295
 296                 self.monitor_lock.release()
 297
 298                 logging.info('Stopped monitoring: {2} on {0}:{1}'.format(vnf_name, vnf_interface, metric))
 299                 return 'Stopped monitoring: {2} on {0}:{1}'.format(vnf_name, vnf_interface, metric)
 300
 301             # delete everything from this vnf
 302             elif metric_dict['vnf_name'] == vnf_name and vnf_interface is None and metric is None:
 303                 self.monitor_lock.acquire()
 304                 self.network_metrics.remove(metric_dict)
 305                 for collector in self.registry._collectors:
 306                     collector_dict = collector._metrics.copy()
 307                     for name, interface, id in collector_dict:
 308                         if name == vnf_name:
 309                             logging.info('3 name:{0} labels:{1} metrics:{2}'.format(collector._name, collector._labelnames,
 310                                                                            collector._metrics))
 311                             collector.remove(name, interface, 'None')
 312
 313                 delete_from_gateway(self.pushgateway, job='sonemu-SDNcontroller')
 314                 self.monitor_lock.release()
 315                 logging.info('Stopped monitoring vnf: {0}'.format(vnf_name))
 316                 return 'Stopped monitoring: {0}'.format(vnf_name)
 317
 318         return 'Error stopping monitoring metric: {0} on {1}:{2}'.format(metric, vnf_name, vnf_interface)
 319
 320
 321 # get all metrics defined in the list and export it to Prometheus
 322     def get_flow_metrics(self):
 323         while self.start_monitoring:
 324
 325             self.monitor_flow_lock.acquire()
 326
 327             for flow_dict in self.flow_metrics:
 328                 data = {}
 329
 330                 data['cookie'] = flow_dict['cookie']
 331                 data['cookie_mask'] = COOKIE_MASK
 332
 333                 if 'tx' in flow_dict['metric_key']:
 334                     data['match'] = {'in_port':flow_dict['mon_port']}
 335                 elif 'rx' in flow_dict['metric_key']:
 336                     data['out_port'] = flow_dict['mon_port']
 337
 338
 339                 # query Ryu
 340                 ret = self.net.ryu_REST('stats/flow', dpid=flow_dict['switch_dpid'], data=data)
 341                 if isinstance(ret, dict):
 342                     flow_stat_dict = ret
 343                 elif isinstance(ret, basestring):
 344                     flow_stat_dict = ast.literal_eval(ret.rstrip())
 345                 else:
 346                     flow_stat_dict = None
 347
 348                 logging.debug('received flow stat:{0} '.format(flow_stat_dict))
 349
 350                 self.set_flow_metric(flow_dict, flow_stat_dict)
 351
 352
 353             try:
 354                 if len(self.flow_metrics) > 0:
 355                     pushadd_to_gateway(self.pushgateway, job='sonemu-SDNcontroller', registry=self.registry)
 356             except Exception, e:
 357                 logging.warning("Pushgateway not reachable: {0} {1}".format(Exception, e))
 358
 359             self.monitor_flow_lock.release()
 360             time.sleep(1)
 361
 362     def get_network_metrics(self):
 363         while self.start_monitoring:
 364
 365             self.monitor_lock.acquire()
 366
 367             # group metrics by dpid to optimize the rest api calls
 368             dpid_list = [metric_dict['switch_dpid'] for metric_dict in self.network_metrics]
 369             dpid_set = set(dpid_list)
 370
 371             for dpid in dpid_set:
 372
 373                 # query Ryu
 374                 ret = self.net.ryu_REST('stats/port', dpid=dpid)
 375                 if isinstance(ret, dict):
 376                     port_stat_dict = ret
 377                 elif isinstance(ret, basestring):
 378                     port_stat_dict = ast.literal_eval(ret.rstrip())
 379                 else:
 380                     port_stat_dict = None
 381
 382                 metric_list = [metric_dict for metric_dict in self.network_metrics
 383                                if int(metric_dict['switch_dpid'])==int(dpid)]
 384
 385                 for metric_dict in metric_list:
 386                     self.set_network_metric(metric_dict, port_stat_dict)
 387
 388             try:
 389                 if len(self.network_metrics) > 0:
 390                     pushadd_to_gateway(self.pushgateway, job='sonemu-SDNcontroller', registry=self.registry)
 391             except Exception, e:
 392                 logging.warning("Pushgateway not reachable: {0} {1}".format(Exception, e))
 393
 394             self.monitor_lock.release()
 395             time.sleep(1)
 396
 397     # add metric to the list to export to Prometheus, parse the Ryu port-stats reply
 398     def set_network_metric(self, metric_dict, port_stat_dict):
 399         # vnf tx is the datacenter switch rx and vice-versa
 400         metric_key = self.switch_tx_rx(metric_dict['metric_key'])
 401         switch_dpid = metric_dict['switch_dpid']
 402         vnf_name = metric_dict['vnf_name']
 403         vnf_interface = metric_dict['vnf_interface']
 404         previous_measurement = metric_dict['previous_measurement']
 405         previous_monitor_time = metric_dict['previous_monitor_time']
 406         mon_port = metric_dict['mon_port']
 407         for port_stat in port_stat_dict[str(switch_dpid)]:
 408             # ovs output also gives back 'LOCAL' port
 409             if port_stat['port_no'] == 'LOCAL':
 410                 continue
 411             if int(port_stat['port_no']) == int(mon_port):
 412                 port_uptime = port_stat['duration_sec'] + port_stat['duration_nsec'] * 10 ** (-9)
 413                 this_measurement = int(port_stat[metric_key])
 414
 415                 # set prometheus metric
 416                 self.prom_metrics[metric_dict['metric_key']].\
 417                     labels(vnf_name=vnf_name, vnf_interface=vnf_interface, flow_id=None).\
 418                     set(this_measurement)
 419
 420                 # also the rate is calculated here, but not used for now
 421                 # (rate can be easily queried from prometheus also)
 422                 if previous_monitor_time <= 0 or previous_monitor_time >= port_uptime:
 423                     metric_dict['previous_measurement'] = int(port_stat[metric_key])
 424                     metric_dict['previous_monitor_time'] = port_uptime
 425                     # do first measurement
 426                     #time.sleep(1)
 427                     #self.monitor_lock.release()
 428                     # rate cannot be calculated yet (need a first measurement)
 429                     metric_rate = None
 430
 431                 else:
 432                     time_delta = (port_uptime - metric_dict['previous_monitor_time'])
 433                     metric_rate = (this_measurement - metric_dict['previous_measurement']) / float(time_delta)
 434
 435                 metric_dict['previous_measurement'] = this_measurement
 436                 metric_dict['previous_monitor_time'] = port_uptime
 437                 return
 438
 439         logging.exception('metric {0} not found on {1}:{2}'.format(metric_key, vnf_name, vnf_interface))
 440         logging.exception('monport:{0}, dpid:{1}'.format(mon_port, switch_dpid))
 441         logging.exception('port dict:{0}'.format(port_stat_dict))
 442         return 'metric {0} not found on {1}:{2}'.format(metric_key, vnf_name, vnf_interface)
 443
 444     def set_flow_metric(self, metric_dict, flow_stat_dict):
 445         # vnf tx is the datacenter switch rx and vice-versa
 446         metric_key = metric_dict['metric_key']
 447         switch_dpid = metric_dict['switch_dpid']
 448         vnf_name = metric_dict['vnf_name']
 449         vnf_interface = metric_dict['vnf_interface']
 450         previous_measurement = metric_dict['previous_measurement']
 451         previous_monitor_time = metric_dict['previous_monitor_time']
 452         cookie = metric_dict['cookie']
 453
 454         counter = 0
 455         for flow_stat in flow_stat_dict[str(switch_dpid)]:
 456             if 'bytes' in metric_key:
 457                 counter += flow_stat['byte_count']
 458             elif 'packet' in metric_key:
 459                 counter += flow_stat['packet_count']
 460
 461         # flow_uptime disabled for now (can give error)
 462         #flow_stat = flow_stat_dict[str(switch_dpid)][0]
 463         #flow_uptime = flow_stat['duration_sec'] + flow_stat['duration_nsec'] * 10 ** (-9)
 464
 465         self.prom_metrics[metric_dict['metric_key']]. \
 466             labels(vnf_name=vnf_name, vnf_interface=vnf_interface, flow_id=cookie). \
 467             set(counter)
 468
 469     def start_Prometheus(self, port=9090):
 470         # prometheus.yml configuration file is located in the same directory as this file
 471         cmd = ["docker",
 472                "run",
 473                "--rm",
 474                "-p", "{0}:9090".format(port),
 475                "-v", "{0}/prometheus.yml:/etc/prometheus/prometheus.yml".format(os.path.dirname(os.path.abspath(__file__))),
 476                "-v", "{0}/profile.rules:/etc/prometheus/profile.rules".format(os.path.dirname(os.path.abspath(__file__))),
 477                "--name", "prometheus",
 478                "prom/prometheus"
 479                ]
 480         logging.info('Start Prometheus container {0}'.format(cmd))
 481         return Popen(cmd)
 482
 483     def start_PushGateway(self, port=PUSHGATEWAY_PORT):
 484         cmd = ["docker",
 485                "run",
 486                "-d",
 487                "-p", "{0}:9091".format(port),
 488                "--name", "pushgateway",
 489                "--label", 'com.containernet=""',
 490                "prom/pushgateway"
 491                ]
 492
 493         logging.info('Start Prometheus Push Gateway container {0}'.format(cmd))
 494         return Popen(cmd)
 495
 496     def start_cAdvisor(self, port=CADVISOR_PORT):
 497         cmd = ["docker",
 498                "run",
 499                "--rm",
 500                "--volume=/:/rootfs:ro",
 501                "--volume=/var/run:/var/run:rw",
 502                "--volume=/sys:/sys:ro",
 503                "--volume=/var/lib/docker/:/var/lib/docker:ro",
 504                "--publish={0}:8080".format(port),
 505                "--name=cadvisor",
 506                "--label",'com.containernet=""',
 507                "google/cadvisor:latest",
 508                "--storage_duration=1m0s",
 509                "--allow_dynamic_housekeeping=true",
 510                #"--housekeeping_interval=1s",
 511                ]
 512         logging.info('Start cAdvisor container {0}'.format(cmd))
 513         return Popen(cmd)
 514
 515     def stop(self):
 516         # stop the monitoring thread
 517         self.start_monitoring = False
 518         self.monitor_thread.join()
 519         self.monitor_flow_thread.join()
 520
 521         # these containers are used for monitoring but are started now outside of son-emu
 522
 523         if self.pushgateway_process is not None:
 524             logging.info('stopping pushgateway container')
 525             self._stop_container('pushgateway')
 526
 527         if self.cadvisor_process is not None:
 528             logging.info('stopping cadvisor container')
 529             self._stop_container('cadvisor')
 530
 531     def switch_tx_rx(self,metric=''):
 532         # when monitoring vnfs, the tx of the datacenter switch is actually the rx of the vnf
 533         # so we need to change the metric name to be consistent with the vnf rx or tx
 534         if 'tx' in metric:
 535             metric = metric.replace('tx','rx')
 536         elif 'rx' in metric:
 537             metric = metric.replace('rx','tx')
 538
 539         return metric
 540
 541     def _stop_container(self, name):
 542
 543         #container = self.dockercli.containers.get(name)
 544         #container.stop()
 545         #container.remove(force=True)
 546
 547         # the only robust way to stop these containers is via Popen, it seems
 548         time.sleep(1)
 549         cmd = ['docker', 'rm', '-f', name]
 550         Popen(cmd)
 551
 552
 553     def update_skewmon(self, vnf_name, resource_name, action):
 554
 555         ret = ''
 556
 557         config_file_path = '/tmp/skewmon.cfg'
 558         configfile = open(config_file_path, 'a+')
 559         try:
 560             config = json.load(configfile)
 561         except:
 562             #not a valid json file or empty
 563             config = {}
 564
 565         #initialize config file
 566         if len(self.skewmon_metrics) == 0:
 567             config = {}
 568         json.dump(config, configfile)
 569         configfile.close()
 570
 571         docker_name = 'mn.' + vnf_name
 572         vnf_container = self.dockercli.containers.get(docker_name)
 573         key = resource_name + '_' + vnf_container.short_id
 574         vnf_id = vnf_container.id
 575
 576         if action == 'start':
 577             # add a new vnf to monitor
 578             config[key] = dict(VNF_NAME=vnf_name,
 579                                 VNF_ID=vnf_id,
 580                                 VNF_METRIC=resource_name)
 581             ret = 'adding to skewness monitor: {0} {1} '.format(vnf_name, resource_name)
 582             logging.info(ret)
 583         elif action == 'stop':
 584             # remove vnf to monitor
 585             config.pop(key)
 586             ret = 'removing from skewness monitor: {0} {1} '.format(vnf_name, resource_name)
 587             logging.info(ret)
 588
 589         self.skewmon_metrics = config
 590         configfile = open(config_file_path, 'w')
 591         json.dump(config, configfile)
 592         configfile.close()
 593
 594         try:
 595             skewmon_container = self.dockercli.containers.get('skewmon')
 596
 597             # remove container if config is empty
 598             if len(config) == 0:
 599                 ret += 'stopping skewness monitor'
 600                 logging.info('stopping skewness monitor')
 601                 skewmon_container.remove(force=True)
 602
 603         except docker.errors.NotFound:
 604             # start container if not running
 605             ret += 'starting skewness monitor'
 606             logging.info('starting skewness monitor')
 607             volumes = {'/sys/fs/cgroup':{'bind':'/sys/fs/cgroup', 'mode':'ro'},
 608                        '/tmp/skewmon.cfg':{'bind':'/config.txt', 'mode':'ro'}}
 609             self.dockercli.containers.run('skewmon',
 610                                           detach=True,
 611                                           volumes=volumes,
 612                                           labels=['com.containernet'],
 613                                           name='skewmon'
 614                                           )
 615             # Wait a while for containers to be completely started
 616             started = False
 617             wait_time = 0
 618             while not started:
 619                 list1 = self.dockercli.containers.list(filters={'status': 'running', 'name': 'prometheus'})
 620                 if len(list1) >= 1:
 621                     time.sleep(1)
 622                     started = True
 623                 if wait_time > 5:
 624                     return 'skewmon not started'
 625                 time.sleep(1)
 626                 wait_time += 1
 627         return ret
 628
 629
 630
 631
 632