src/emuvim/dcemulator/monitoring.py

   1 # Copyright (c) 2015 SONATA-NFV and Paderborn University
   2 # ALL RIGHTS RESERVED.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #    http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 #
  16 # Neither the name of the SONATA-NFV, Paderborn University
  17 # nor the names of its contributors may be used to endorse or promote
  18 # products derived from this software without specific prior written
  19 # permission.
  20 #
  21 # This work has been performed in the framework of the SONATA project,
  22 # funded by the European Commission under Grant number 671517 through
  23 # the Horizon 2020 and 5G-PPP programmes. The authors would like to
  24 # acknowledge the contributions of their colleagues of the SONATA
  25 # partner consortium (www.sonata-nfv.eu).
  26 import logging
  27 from mininet.node import OVSSwitch
  28 import ast
  29 import time
  30 from prometheus_client import Gauge, CollectorRegistry, \
  31     pushadd_to_gateway, delete_from_gateway
  32 import threading
  33 from subprocess import Popen
  34 import os
  35 import docker
  36 import json
  37 from copy import deepcopy
  38
  39 logging.basicConfig()
  40
  41 """
  42 class to read openflow stats from the Ryu controller of the DCNetwork
  43 """
  44
  45 PUSHGATEWAY_PORT = 9091
  46 # we cannot use port 8080 because ryu-ofrest api  is already using that one
  47 CADVISOR_PORT = 8081
  48
  49 COOKIE_MASK = 0xffffffff
  50
  51
  52 class DCNetworkMonitor():
  53     def __init__(self, net):
  54         self.net = net
  55         self.dockercli = docker.from_env()
  56
  57         # pushgateway address
  58         self.pushgateway = 'localhost:{0}'.format(PUSHGATEWAY_PORT)
  59
  60         # supported Prometheus metrics
  61         self.registry = CollectorRegistry()
  62         self.prom_tx_packet_count = Gauge('sonemu_tx_count_packets', 'Total number of packets sent',
  63                                           ['vnf_name', 'vnf_interface', 'flow_id'], registry=self.registry)
  64         self.prom_rx_packet_count = Gauge('sonemu_rx_count_packets', 'Total number of packets received',
  65                                           ['vnf_name', 'vnf_interface', 'flow_id'], registry=self.registry)
  66         self.prom_tx_byte_count = Gauge('sonemu_tx_count_bytes', 'Total number of bytes sent',
  67                                         ['vnf_name', 'vnf_interface', 'flow_id'], registry=self.registry)
  68         self.prom_rx_byte_count = Gauge('sonemu_rx_count_bytes', 'Total number of bytes received',
  69                                         ['vnf_name', 'vnf_interface', 'flow_id'], registry=self.registry)
  70
  71         self.prom_metrics = {'tx_packets': self.prom_tx_packet_count, 'rx_packets': self.prom_rx_packet_count,
  72                              'tx_bytes': self.prom_tx_byte_count, 'rx_bytes': self.prom_rx_byte_count}
  73
  74         # list of installed metrics to monitor
  75         # each entry can contain this data
  76         '''
  77         {
  78         switch_dpid = 0
  79         vnf_name = None
  80         vnf_interface = None
  81         previous_measurement = 0
  82         previous_monitor_time = 0
  83         metric_key = None
  84         mon_port = None
  85         }
  86         '''
  87         self.monitor_lock = threading.Lock()
  88         self.monitor_flow_lock = threading.Lock()
  89         self.network_metrics = []
  90         self.flow_metrics = []
  91         self.skewmon_metrics = {}
  92
  93         # start monitoring thread
  94         self.start_monitoring = True
  95         self.monitor_thread = threading.Thread(target=self.get_network_metrics)
  96         self.monitor_thread.start()
  97
  98         self.monitor_flow_thread = threading.Thread(
  99             target=self.get_flow_metrics)
 100         self.monitor_flow_thread.start()
 101
 102         # helper tools
 103         # cAdvisor, Prometheus pushgateway are started as external container,
 104         # to gather monitoring metric in son-emu
 105         self.pushgateway_process = self.start_PushGateway()
 106         self.cadvisor_process = self.start_cAdvisor()
 107
 108     # first set some parameters, before measurement can start
 109
 110     def setup_flow(self, vnf_name, vnf_interface=None,
 111                    metric='tx_packets', cookie=0):
 112
 113         flow_metric = {}
 114
 115         # check if port is specified (vnf:port)
 116         if vnf_interface is None:
 117             # take first interface by default
 118             connected_sw = self.net.DCNetwork_graph.neighbors(vnf_name)[0]
 119             link_dict = self.net.DCNetwork_graph[vnf_name][connected_sw]
 120             vnf_interface = link_dict[0]['src_port_id']
 121
 122         flow_metric['vnf_name'] = vnf_name
 123         flow_metric['vnf_interface'] = vnf_interface
 124
 125         vnf_switch = None
 126         for connected_sw in self.net.DCNetwork_graph.neighbors(vnf_name):
 127             link_dict = self.net.DCNetwork_graph[vnf_name][connected_sw]
 128             for link in link_dict:
 129                 if link_dict[link]['src_port_id'] == vnf_interface:
 130                     # found the right link and connected switch
 131                     vnf_switch = connected_sw
 132                     flow_metric['mon_port'] = link_dict[link]['dst_port_nr']
 133                     break
 134
 135         if not vnf_switch:
 136             logging.exception("vnf switch of {0}:{1} not found!".format(
 137                 vnf_name, vnf_interface))
 138             return "vnf switch of {0}:{1} not found!".format(
 139                 vnf_name, vnf_interface)
 140
 141         try:
 142             # default port direction to monitor
 143             if metric is None:
 144                 metric = 'tx_packets'
 145
 146             next_node = self.net.getNodeByName(vnf_switch)
 147
 148             if not isinstance(next_node, OVSSwitch):
 149                 logging.info(
 150                     "vnf: {0} is not connected to switch".format(vnf_name))
 151                 return
 152
 153             flow_metric['previous_measurement'] = 0
 154             flow_metric['previous_monitor_time'] = 0
 155
 156             flow_metric['switch_dpid'] = int(str(next_node.dpid), 16)
 157             flow_metric['metric_key'] = metric
 158             flow_metric['cookie'] = cookie
 159
 160             self.monitor_flow_lock.acquire()
 161             self.flow_metrics.append(flow_metric)
 162             self.monitor_flow_lock.release()
 163
 164             logging.info('Started monitoring flow:{3} {2} on {0}:{1}'.format(
 165                 vnf_name, vnf_interface, metric, cookie))
 166             return 'Started monitoring flow:{3} {2} on {0}:{1}'.format(
 167                 vnf_name, vnf_interface, metric, cookie)
 168
 169         except Exception as ex:
 170             logging.exception("setup_metric error.")
 171             return ex.message
 172
 173     def stop_flow(self, vnf_name, vnf_interface=None, metric=None, cookie=0,):
 174
 175         # check if port is specified (vnf:port)
 176         if vnf_interface is None and metric is not None:
 177             # take first interface by default
 178             connected_sw = self.net.DCNetwork_graph.neighbors(vnf_name)[0]
 179             link_dict = self.net.DCNetwork_graph[vnf_name][connected_sw]
 180             vnf_interface = link_dict[0]['src_port_id']
 181
 182         for flow_dict in self.flow_metrics:
 183             if flow_dict['vnf_name'] == vnf_name and flow_dict['vnf_interface'] == vnf_interface \
 184                     and flow_dict['metric_key'] == metric and flow_dict['cookie'] == cookie:
 185
 186                 self.monitor_flow_lock.acquire()
 187
 188                 self.flow_metrics.remove(flow_dict)
 189
 190                 # set metric to NaN
 191                 self.prom_metrics[flow_dict['metric_key']]. \
 192                     labels(vnf_name=vnf_name, vnf_interface=vnf_interface, flow_id=cookie). \
 193                     set(float('nan'))
 194
 195                 delete_from_gateway(
 196                     self.pushgateway, job='sonemu-SDNcontroller')
 197
 198                 self.monitor_flow_lock.release()
 199
 200                 logging.info('Stopped monitoring flow {3}: {2} on {0}:{1}'.format(
 201                     vnf_name, vnf_interface, metric, cookie))
 202                 return 'Stopped monitoring flow {3}: {2} on {0}:{1}'.format(
 203                     vnf_name, vnf_interface, metric, cookie)
 204
 205         return 'Error stopping monitoring flow: {0} on {1}:{2}'.format(
 206             metric, vnf_name, vnf_interface)
 207
 208     # first set some parameters, before measurement can start
 209
 210     def setup_metric(self, vnf_name, vnf_interface=None, metric='tx_packets'):
 211
 212         network_metric = {}
 213
 214         # check if port is specified (vnf:port)
 215         if vnf_interface is None or vnf_interface == '':
 216             # take first interface by default
 217             connected_sw = self.net.DCNetwork_graph.neighbors(vnf_name)[0]
 218             link_dict = self.net.DCNetwork_graph[vnf_name][connected_sw]
 219             vnf_interface = link_dict[0]['src_port_id']
 220
 221         network_metric['vnf_name'] = vnf_name
 222         network_metric['vnf_interface'] = vnf_interface
 223
 224         for connected_sw in self.net.DCNetwork_graph.neighbors(vnf_name):
 225             link_dict = self.net.DCNetwork_graph[vnf_name][connected_sw]
 226             for link in link_dict:
 227                 if link_dict[link]['src_port_id'] == vnf_interface:
 228                     # found the right link and connected switch
 229                     network_metric['mon_port'] = link_dict[link]['dst_port_nr']
 230                     break
 231
 232         if 'mon_port' not in network_metric:
 233             logging.exception("vnf interface {0}:{1} not found!".format(
 234                 vnf_name, vnf_interface))
 235             return "vnf interface {0}:{1} not found!".format(
 236                 vnf_name, vnf_interface)
 237
 238         try:
 239             # default port direction to monitor
 240             if metric is None:
 241                 metric = 'tx_packets'
 242
 243             vnf_switch = self.net.DCNetwork_graph.neighbors(str(vnf_name))
 244
 245             if len(vnf_switch) > 1:
 246                 logging.info("vnf: {0} has multiple ports".format(vnf_name))
 247                 return
 248             elif len(vnf_switch) == 0:
 249                 logging.info("vnf: {0} is not connected".format(vnf_name))
 250                 return
 251             else:
 252                 vnf_switch = vnf_switch[0]
 253             next_node = self.net.getNodeByName(vnf_switch)
 254
 255             if not isinstance(next_node, OVSSwitch):
 256                 logging.info(
 257                     "vnf: {0} is not connected to switch".format(vnf_name))
 258                 return
 259
 260             network_metric['previous_measurement'] = 0
 261             network_metric['previous_monitor_time'] = 0
 262
 263             network_metric['switch_dpid'] = int(str(next_node.dpid), 16)
 264             network_metric['metric_key'] = metric
 265
 266             self.monitor_lock.acquire()
 267             self.network_metrics.append(network_metric)
 268             self.monitor_lock.release()
 269
 270             logging.info('Started monitoring: {2} on {0}:{1}'.format(
 271                 vnf_name, vnf_interface, metric))
 272             return 'Started monitoring: {2} on {0}:{1}'.format(
 273                 vnf_name, vnf_interface, metric)
 274
 275         except Exception as ex:
 276             logging.exception("setup_metric error.")
 277             return ex.message
 278
 279     def stop_metric(self, vnf_name, vnf_interface=None, metric=None):
 280
 281         # check if port is specified (vnf:port)
 282         if vnf_interface is None and metric is not None:
 283             # take first interface by default
 284             connected_sw = self.net.DCNetwork_graph.neighbors(vnf_name)[0]
 285             link_dict = self.net.DCNetwork_graph[vnf_name][connected_sw]
 286             vnf_interface = link_dict[0]['src_port_id']
 287
 288         for metric_dict in deepcopy(self.network_metrics):
 289             if metric_dict['vnf_name'] == vnf_name and metric_dict['vnf_interface'] == vnf_interface \
 290                     and metric_dict['metric_key'] == metric:
 291
 292                 self.monitor_lock.acquire()
 293
 294                 self.network_metrics.remove(metric_dict)
 295
 296                 # set values to NaN, prometheus api currently does not support removal of metrics
 297                 # self.prom_metrics[metric_dict['metric_key']].labels(vnf_name, vnf_interface).set(float('nan'))
 298                 self.prom_metrics[metric_dict['metric_key']]. \
 299                     labels(vnf_name=vnf_name, vnf_interface=vnf_interface, flow_id=None). \
 300                     set(float('nan'))
 301
 302                 # this removes the complete metric, all labels...
 303                 # 1 single monitor job for all metrics of the SDN controller
 304                 # we can only  remove from the pushgateway grouping keys(labels) which we have defined for the add_to_pushgateway
 305                 # we can not specify labels from the metrics to be removed
 306                 # if we need to remove the metrics seperatelty, we need to give
 307                 # them a separate grouping key, and probably a diffferent
 308                 # registry also
 309                 delete_from_gateway(
 310                     self.pushgateway, job='sonemu-SDNcontroller')
 311
 312                 self.monitor_lock.release()
 313
 314                 logging.info('Stopped monitoring: {2} on {0}:{1}'.format(
 315                     vnf_name, vnf_interface, metric))
 316                 return 'Stopped monitoring: {2} on {0}:{1}'.format(
 317                     vnf_name, vnf_interface, metric)
 318
 319             # delete everything from this vnf
 320             elif metric_dict['vnf_name'] == vnf_name and vnf_interface is None and metric is None:
 321                 self.monitor_lock.acquire()
 322                 self.network_metrics.remove(metric_dict)
 323                 logging.info('remove metric from monitor: vnf_name:{0} vnf_interface:{1} mon_port:{2}'.format(
 324                     metric_dict['vnf_name'], metric_dict['vnf_interface'], metric_dict['mon_port']))
 325
 326                 delete_from_gateway(
 327                     self.pushgateway, job='sonemu-SDNcontroller')
 328                 self.monitor_lock.release()
 329                 continue
 330
 331         if vnf_interface is None and metric is None:
 332             logging.info('Stopped monitoring vnf: {0}'.format(vnf_name))
 333             return 'Stopped monitoring: {0}'.format(vnf_name)
 334         else:
 335             return 'Error stopping monitoring metric: {0} on {1}:{2}'.format(
 336                 metric, vnf_name, vnf_interface)
 337
 338
 339 # get all metrics defined in the list and export it to Prometheus
 340
 341     def get_flow_metrics(self):
 342         while self.start_monitoring:
 343
 344             self.monitor_flow_lock.acquire()
 345
 346             for flow_dict in self.flow_metrics:
 347                 data = {}
 348
 349                 data['cookie'] = flow_dict['cookie']
 350                 data['cookie_mask'] = COOKIE_MASK
 351
 352                 if 'tx' in flow_dict['metric_key']:
 353                     data['match'] = {'in_port': flow_dict['mon_port']}
 354                 elif 'rx' in flow_dict['metric_key']:
 355                     data['out_port'] = flow_dict['mon_port']
 356
 357                 # query Ryu
 358                 ret = self.net.ryu_REST(
 359                     'stats/flow', dpid=flow_dict['switch_dpid'], data=data)
 360                 if isinstance(ret, dict):
 361                     flow_stat_dict = ret
 362                 elif isinstance(ret, basestring):
 363                     flow_stat_dict = ast.literal_eval(ret.rstrip())
 364                 else:
 365                     flow_stat_dict = None
 366
 367                 logging.debug('received flow stat:{0} '.format(flow_stat_dict))
 368
 369                 self.set_flow_metric(flow_dict, flow_stat_dict)
 370
 371             try:
 372                 if len(self.flow_metrics) > 0:
 373                     pushadd_to_gateway(
 374                         self.pushgateway, job='sonemu-SDNcontroller', registry=self.registry)
 375             except Exception as e:
 376                 logging.warning(
 377                     "Pushgateway not reachable: {0} {1}".format(Exception, e))
 378
 379             self.monitor_flow_lock.release()
 380             time.sleep(1)
 381
 382     def get_network_metrics(self):
 383         while self.start_monitoring:
 384
 385             self.monitor_lock.acquire()
 386
 387             # group metrics by dpid to optimize the rest api calls
 388             dpid_list = [metric_dict['switch_dpid']
 389                          for metric_dict in self.network_metrics]
 390             dpid_set = set(dpid_list)
 391
 392             for dpid in dpid_set:
 393
 394                 # query Ryu
 395                 ret = self.net.ryu_REST('stats/port', dpid=dpid)
 396                 if isinstance(ret, dict):
 397                     port_stat_dict = ret
 398                 elif isinstance(ret, basestring):
 399                     port_stat_dict = ast.literal_eval(ret.rstrip())
 400                 else:
 401                     port_stat_dict = None
 402
 403                 metric_list = [metric_dict for metric_dict in self.network_metrics
 404                                if int(metric_dict['switch_dpid']) == int(dpid)]
 405
 406                 for metric_dict in metric_list:
 407                     self.set_network_metric(metric_dict, port_stat_dict)
 408
 409             try:
 410                 if len(self.network_metrics) > 0:
 411                     pushadd_to_gateway(
 412                         self.pushgateway, job='sonemu-SDNcontroller', registry=self.registry)
 413             except Exception as e:
 414                 logging.warning(
 415                     "Pushgateway not reachable: {0} {1}".format(Exception, e))
 416
 417             self.monitor_lock.release()
 418             time.sleep(1)
 419
 420     # add metric to the list to export to Prometheus, parse the Ryu port-stats
 421     # reply
 422     def set_network_metric(self, metric_dict, port_stat_dict):
 423         # vnf tx is the datacenter switch rx and vice-versa
 424         metric_key = self.switch_tx_rx(metric_dict['metric_key'])
 425         switch_dpid = metric_dict['switch_dpid']
 426         vnf_name = metric_dict['vnf_name']
 427         vnf_interface = metric_dict['vnf_interface']
 428         previous_monitor_time = metric_dict['previous_monitor_time']
 429         mon_port = metric_dict['mon_port']
 430         for port_stat in port_stat_dict[str(switch_dpid)]:
 431             # ovs output also gives back 'LOCAL' port
 432             if port_stat['port_no'] == 'LOCAL':
 433                 continue
 434             if int(port_stat['port_no']) == int(mon_port):
 435                 port_uptime = port_stat['duration_sec'] + \
 436                     port_stat['duration_nsec'] * 10 ** (-9)
 437                 this_measurement = int(port_stat[metric_key])
 438
 439                 # set prometheus metric
 440                 self.prom_metrics[metric_dict['metric_key']].\
 441                     labels(vnf_name=vnf_name, vnf_interface=vnf_interface, flow_id=None).\
 442                     set(this_measurement)
 443
 444                 # also the rate is calculated here, but not used for now
 445                 # (rate can be easily queried from prometheus also)
 446                 if previous_monitor_time <= 0 or previous_monitor_time >= port_uptime:
 447                     metric_dict['previous_measurement'] = int(
 448                         port_stat[metric_key])
 449                     metric_dict['previous_monitor_time'] = port_uptime
 450                     # do first measurement
 451                     # time.sleep(1)
 452                     # self.monitor_lock.release()
 453                     # rate cannot be calculated yet (need a first measurement)
 454                 metric_dict['previous_measurement'] = this_measurement
 455                 metric_dict['previous_monitor_time'] = port_uptime
 456                 return
 457
 458         logging.exception('metric {0} not found on {1}:{2}'.format(
 459             metric_key, vnf_name, vnf_interface))
 460         logging.exception(
 461             'monport:{0}, dpid:{1}'.format(mon_port, switch_dpid))
 462         logging.exception(
 463             'monitored network_metrics:{0}'.format(self.network_metrics))
 464         logging.exception('port dict:{0}'.format(port_stat_dict))
 465         return 'metric {0} not found on {1}:{2}'.format(
 466             metric_key, vnf_name, vnf_interface)
 467
 468     def set_flow_metric(self, metric_dict, flow_stat_dict):
 469         # vnf tx is the datacenter switch rx and vice-versa
 470         metric_key = metric_dict['metric_key']
 471         switch_dpid = metric_dict['switch_dpid']
 472         vnf_name = metric_dict['vnf_name']
 473         vnf_interface = metric_dict['vnf_interface']
 474         cookie = metric_dict['cookie']
 475
 476         counter = 0
 477         for flow_stat in flow_stat_dict[str(switch_dpid)]:
 478             if 'bytes' in metric_key:
 479                 counter += flow_stat['byte_count']
 480             elif 'packet' in metric_key:
 481                 counter += flow_stat['packet_count']
 482
 483         # flow_uptime disabled for now (can give error)
 484         # flow_stat = flow_stat_dict[str(switch_dpid)][0]
 485         # flow_uptime = flow_stat['duration_sec'] + flow_stat['duration_nsec'] * 10 ** (-9)
 486
 487         self.prom_metrics[metric_dict['metric_key']]. \
 488             labels(vnf_name=vnf_name, vnf_interface=vnf_interface, flow_id=cookie). \
 489             set(counter)
 490
 491     def start_Prometheus(self, port=9090):
 492         # prometheus.yml configuration file is located in the same directory as
 493         # this file
 494         cmd = ["docker",
 495                "run",
 496                "--rm",
 497                "-p", "{0}:9090".format(port),
 498                "-v", "{0}/prometheus.yml:/etc/prometheus/prometheus.yml".format(
 499                    os.path.dirname(os.path.abspath(__file__))),
 500                "-v", "{0}/profile.rules:/etc/prometheus/profile.rules".format(
 501                    os.path.dirname(os.path.abspath(__file__))),
 502                "--name", "prometheus",
 503                "prom/prometheus"
 504                ]
 505         logging.info('Start Prometheus container {0}'.format(cmd))
 506         return Popen(cmd)
 507
 508     def start_PushGateway(self, port=PUSHGATEWAY_PORT):
 509         cmd = ["docker",
 510                "run",
 511                "-d",
 512                "-p", "{0}:9091".format(port),
 513                "--name", "pushgateway",
 514                "--label", 'com.containernet=""',
 515                "prom/pushgateway"
 516                ]
 517
 518         logging.info('Start Prometheus Push Gateway container {0}'.format(cmd))
 519         return Popen(cmd)
 520
 521     def start_cAdvisor(self, port=CADVISOR_PORT):
 522         cmd = ["docker",
 523                "run",
 524                "--rm",
 525                "--volume=/:/rootfs:ro",
 526                "--volume=/var/run:/var/run:rw",
 527                "--volume=/sys:/sys:ro",
 528                "--volume=/var/lib/docker/:/var/lib/docker:ro",
 529                "--publish={0}:8080".format(port),
 530                "--name=cadvisor",
 531                "--label", 'com.containernet=""',
 532                "--detach=true",
 533                "google/cadvisor:latest",
 534                # "--storage_duration=1m0s",
 535                # "--allow_dynamic_housekeeping=true",
 536                # "--housekeeping_interval=1s",
 537                ]
 538         logging.info('Start cAdvisor container {0}'.format(cmd))
 539         return Popen(cmd)
 540
 541     def stop(self):
 542         # stop the monitoring thread
 543         self.start_monitoring = False
 544         self.monitor_thread.join()
 545         self.monitor_flow_thread.join()
 546
 547         # these containers are used for monitoring but are started now outside
 548         # of son-emu
 549
 550         if self.pushgateway_process is not None:
 551             logging.info('stopping pushgateway container')
 552             self._stop_container('pushgateway')
 553
 554         if self.cadvisor_process is not None:
 555             logging.info('stopping cadvisor container')
 556             self._stop_container('cadvisor')
 557
 558     def switch_tx_rx(self, metric=''):
 559         # when monitoring vnfs, the tx of the datacenter switch is actually the rx of the vnf
 560         # so we need to change the metric name to be consistent with the vnf rx
 561         # or tx
 562         if 'tx' in metric:
 563             metric = metric.replace('tx', 'rx')
 564         elif 'rx' in metric:
 565             metric = metric.replace('rx', 'tx')
 566
 567         return metric
 568
 569     def _stop_container(self, name):
 570
 571         # container = self.dockercli.containers.get(name)
 572         # container.stop()
 573         # container.remove(force=True)
 574
 575         # the only robust way to stop these containers is via Popen, it seems
 576         time.sleep(1)
 577         cmd = ['docker', 'rm', '-f', name]
 578         Popen(cmd)
 579
 580     def update_skewmon(self, vnf_name, resource_name, action):
 581
 582         ret = ''
 583
 584         config_file_path = '/tmp/skewmon.cfg'
 585         configfile = open(config_file_path, 'a+')
 586         try:
 587             config = json.load(configfile)
 588         except BaseException:
 589             # not a valid json file or empty
 590             config = {}
 591
 592         # initialize config file
 593         if len(self.skewmon_metrics) == 0:
 594             config = {}
 595         json.dump(config, configfile)
 596         configfile.close()
 597
 598         docker_name = 'mn.' + vnf_name
 599         vnf_container = self.dockercli.containers.get(docker_name)
 600         key = resource_name + '_' + vnf_container.short_id
 601         vnf_id = vnf_container.id
 602
 603         if action == 'start':
 604             # add a new vnf to monitor
 605             config[key] = dict(VNF_NAME=vnf_name,
 606                                VNF_ID=vnf_id,
 607                                VNF_METRIC=resource_name)
 608             ret = 'adding to skewness monitor: {0} {1} '.format(
 609                 vnf_name, resource_name)
 610             logging.info(ret)
 611         elif action == 'stop':
 612             # remove vnf to monitor
 613             config.pop(key)
 614             ret = 'removing from skewness monitor: {0} {1} '.format(
 615                 vnf_name, resource_name)
 616             logging.info(ret)
 617
 618         self.skewmon_metrics = config
 619         configfile = open(config_file_path, 'w')
 620         json.dump(config, configfile)
 621         configfile.close()
 622
 623         try:
 624             skewmon_container = self.dockercli.containers.get('skewmon')
 625
 626             # remove container if config is empty
 627             if len(config) == 0:
 628                 ret += 'stopping skewness monitor'
 629                 logging.info('stopping skewness monitor')
 630                 skewmon_container.remove(force=True)
 631
 632         except docker.errors.NotFound:
 633             # start container if not running
 634             ret += 'starting skewness monitor'
 635             logging.info('starting skewness monitor')
 636             volumes = {'/sys/fs/cgroup': {'bind': '/sys/fs/cgroup', 'mode': 'ro'},
 637                        '/tmp/skewmon.cfg': {'bind': '/config.txt', 'mode': 'ro'}}
 638             self.dockercli.containers.run('skewmon',
 639                                           detach=True,
 640                                           volumes=volumes,
 641                                           labels=['com.containernet'],
 642                                           name='skewmon'
 643                                           )
 644             # Wait a while for containers to be completely started
 645             started = False
 646             wait_time = 0
 647             while not started:
 648                 list1 = self.dockercli.containers.list(
 649                     filters={'status': 'running', 'name': 'prometheus'})
 650                 if len(list1) >= 1:
 651                     time.sleep(1)
 652                     started = True
 653                 if wait_time > 5:
 654                     return 'skewmon not started'
 655                 time.sleep(1)
 656                 wait_time += 1
 657         return ret
 658
 659     def term(self, vnf_list=[]):
 660         """
 661         Start a terminal window for the specified VNFs
 662         (start a terminal for all VNFs if vnf_list is empty)
 663         :param vnf_list:
 664         :return:
 665         """
 666
 667         if vnf_list is None:
 668             vnf_list = []
 669         if not isinstance(vnf_list, list):
 670             vnf_list = str(vnf_list).split(',')
 671             vnf_list = map(str.strip, vnf_list)
 672         logging.info('vnf_list: {}'.format(vnf_list))
 673
 674         return self.start_xterm(vnf_list)
 675
 676     # start an xterm for the specfified vnfs
 677
 678     def start_xterm(self, vnf_names):
 679         # start xterm for all vnfs
 680         for vnf_name in vnf_names:
 681             terminal_cmd = "docker exec -it mn.{0} /bin/bash".format(vnf_name)
 682
 683             cmd = ['xterm', '-xrm', 'XTerm*selectToClipboard: true', '-xrm', 'XTerm.vt100.allowTitleOps: false',
 684                    '-T', vnf_name,
 685                    '-e', terminal_cmd]
 686             Popen(cmd)
 687
 688         ret = 'xterms started for {0}'.format(vnf_names)
 689         if len(vnf_names) == 0:
 690             ret = 'vnf list is empty, no xterms started'
 691         return ret