src/emuvim/dcemulator/monitoring.py

   1 """
   2 Copyright (c) 2015 SONATA-NFV
   3 ALL RIGHTS RESERVED.
   4
   5 Licensed under the Apache License, Version 2.0 (the "License");
   6 you may not use this file except in compliance with the License.
   7 You may obtain a copy of the License at
   8
   9     http://www.apache.org/licenses/LICENSE-2.0
  10
  11 Unless required by applicable law or agreed to in writing, software
  12 distributed under the License is distributed on an "AS IS" BASIS,
  13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 See the License for the specific language governing permissions and
  15 limitations under the License.
  16
  17 Neither the name of the SONATA-NFV [, ANY ADDITIONAL AFFILIATION]
  18 nor the names of its contributors may be used to endorse or promote
  19 products derived from this software without specific prior written
  20 permission.
  21
  22 This work has been performed in the framework of the SONATA project,
  23 funded by the European Commission under Grant number 671517 through
  24 the Horizon 2020 and 5G-PPP programmes. The authors would like to
  25 acknowledge the contributions of their colleagues of the SONATA
  26 partner consortium (www.sonata-nfv.eu).
  27 """
  28
  29 import logging
  30 import sys
  31 from mininet.node import  OVSSwitch
  32 import ast
  33 import time
  34 from prometheus_client import start_http_server, Summary, Histogram, Gauge, Counter, REGISTRY, CollectorRegistry, \
  35     pushadd_to_gateway, push_to_gateway, delete_from_gateway
  36 import threading
  37 from subprocess import Popen
  38 import os
  39 import docker
  40 import json
  41 from copy import deepcopy
  42
  43 logging.basicConfig()
  44
  45 """
  46 class to read openflow stats from the Ryu controller of the DCNetwork
  47 """
  48
  49 PUSHGATEWAY_PORT = 9091
  50 # we cannot use port 8080 because ryu-ofrest api  is already using that one
  51 CADVISOR_PORT = 8081
  52
  53 COOKIE_MASK = 0xffffffff
  54
  55 class DCNetworkMonitor():
  56     def __init__(self, net):
  57         self.net = net
  58         self.dockercli = docker.from_env()
  59
  60         # pushgateway address
  61         self.pushgateway = 'localhost:{0}'.format(PUSHGATEWAY_PORT)
  62
  63         # supported Prometheus metrics
  64         self.registry = CollectorRegistry()
  65         self.prom_tx_packet_count = Gauge('sonemu_tx_count_packets', 'Total number of packets sent',
  66                                           ['vnf_name', 'vnf_interface', 'flow_id'], registry=self.registry)
  67         self.prom_rx_packet_count = Gauge('sonemu_rx_count_packets', 'Total number of packets received',
  68                                           ['vnf_name', 'vnf_interface', 'flow_id'], registry=self.registry)
  69         self.prom_tx_byte_count = Gauge('sonemu_tx_count_bytes', 'Total number of bytes sent',
  70                                         ['vnf_name', 'vnf_interface', 'flow_id'], registry=self.registry)
  71         self.prom_rx_byte_count = Gauge('sonemu_rx_count_bytes', 'Total number of bytes received',
  72                                         ['vnf_name', 'vnf_interface', 'flow_id'], registry=self.registry)
  73
  74         self.prom_metrics={'tx_packets':self.prom_tx_packet_count, 'rx_packets':self.prom_rx_packet_count,
  75                            'tx_bytes':self.prom_tx_byte_count,'rx_bytes':self.prom_rx_byte_count}
  76
  77         # list of installed metrics to monitor
  78         # each entry can contain this data
  79         '''
  80         {
  81         switch_dpid = 0
  82         vnf_name = None
  83         vnf_interface = None
  84         previous_measurement = 0
  85         previous_monitor_time = 0
  86         metric_key = None
  87         mon_port = None
  88         }
  89         '''
  90         self.monitor_lock = threading.Lock()
  91         self.monitor_flow_lock = threading.Lock()
  92         self.network_metrics = []
  93         self.flow_metrics = []
  94         self.skewmon_metrics = {}
  95
  96         # start monitoring thread
  97         self.start_monitoring = True
  98         self.monitor_thread = threading.Thread(target=self.get_network_metrics)
  99         self.monitor_thread.start()
 100
 101         self.monitor_flow_thread = threading.Thread(target=self.get_flow_metrics)
 102         self.monitor_flow_thread.start()
 103
 104         # helper tools
 105         # cAdvisor, Prometheus pushgateway are started as external container, to gather monitoring metric in son-emu
 106         self.pushgateway_process = self.start_PushGateway()
 107         self.cadvisor_process = self.start_cAdvisor()
 108
 109
 110     # first set some parameters, before measurement can start
 111     def setup_flow(self, vnf_name, vnf_interface=None, metric='tx_packets', cookie=0):
 112
 113         flow_metric = {}
 114
 115         # check if port is specified (vnf:port)
 116         if vnf_interface is None:
 117             # take first interface by default
 118             connected_sw = self.net.DCNetwork_graph.neighbors(vnf_name)[0]
 119             link_dict = self.net.DCNetwork_graph[vnf_name][connected_sw]
 120             vnf_interface = link_dict[0]['src_port_id']
 121
 122         flow_metric['vnf_name'] = vnf_name
 123         flow_metric['vnf_interface'] = vnf_interface
 124
 125         vnf_switch = None
 126         for connected_sw in self.net.DCNetwork_graph.neighbors(vnf_name):
 127             link_dict = self.net.DCNetwork_graph[vnf_name][connected_sw]
 128             for link in link_dict:
 129                 if link_dict[link]['src_port_id'] == vnf_interface:
 130                     # found the right link and connected switch
 131                     vnf_switch = connected_sw
 132                     flow_metric['mon_port'] = link_dict[link]['dst_port_nr']
 133                     break
 134
 135         if not vnf_switch:
 136             logging.exception("vnf switch of {0}:{1} not found!".format(vnf_name, vnf_interface))
 137             return "vnf switch of {0}:{1} not found!".format(vnf_name, vnf_interface)
 138
 139         try:
 140             # default port direction to monitor
 141             if metric is None:
 142                 metric = 'tx_packets'
 143
 144             next_node = self.net.getNodeByName(vnf_switch)
 145
 146             if not isinstance(next_node, OVSSwitch):
 147                 logging.info("vnf: {0} is not connected to switch".format(vnf_name))
 148                 return
 149
 150             flow_metric['previous_measurement'] = 0
 151             flow_metric['previous_monitor_time'] = 0
 152
 153             flow_metric['switch_dpid'] = int(str(next_node.dpid), 16)
 154             flow_metric['metric_key'] = metric
 155             flow_metric['cookie'] = cookie
 156
 157             self.monitor_flow_lock.acquire()
 158             self.flow_metrics.append(flow_metric)
 159             self.monitor_flow_lock.release()
 160
 161             logging.info('Started monitoring flow:{3} {2} on {0}:{1}'.format(vnf_name, vnf_interface, metric, cookie))
 162             return 'Started monitoring flow:{3} {2} on {0}:{1}'.format(vnf_name, vnf_interface, metric, cookie)
 163
 164         except Exception as ex:
 165             logging.exception("setup_metric error.")
 166             return ex.message
 167
 168     def stop_flow(self, vnf_name, vnf_interface=None, metric=None, cookie=0,):
 169
 170         # check if port is specified (vnf:port)
 171         if vnf_interface is None and metric is not None:
 172             # take first interface by default
 173             connected_sw = self.net.DCNetwork_graph.neighbors(vnf_name)[0]
 174             link_dict = self.net.DCNetwork_graph[vnf_name][connected_sw]
 175             vnf_interface = link_dict[0]['src_port_id']
 176
 177         for flow_dict in self.flow_metrics:
 178             if flow_dict['vnf_name'] == vnf_name and flow_dict['vnf_interface'] == vnf_interface \
 179                     and flow_dict['metric_key'] == metric and flow_dict['cookie'] == cookie:
 180
 181                 self.monitor_flow_lock.acquire()
 182
 183                 self.flow_metrics.remove(flow_dict)
 184
 185                 # set metric to NaN
 186                 self.prom_metrics[flow_dict['metric_key']]. \
 187                     labels(vnf_name=vnf_name, vnf_interface=vnf_interface, flow_id=cookie). \
 188                     set(float('nan'))
 189
 190                 delete_from_gateway(self.pushgateway, job='sonemu-SDNcontroller')
 191
 192                 self.monitor_flow_lock.release()
 193
 194                 logging.info('Stopped monitoring flow {3}: {2} on {0}:{1}'.format(vnf_name, vnf_interface, metric, cookie))
 195                 return 'Stopped monitoring flow {3}: {2} on {0}:{1}'.format(vnf_name, vnf_interface, metric, cookie)
 196
 197         return 'Error stopping monitoring flow: {0} on {1}:{2}'.format(metric, vnf_name, vnf_interface)
 198
 199
 200     # first set some parameters, before measurement can start
 201     def setup_metric(self, vnf_name, vnf_interface=None, metric='tx_packets'):
 202
 203         network_metric = {}
 204
 205         # check if port is specified (vnf:port)
 206         if vnf_interface is None or vnf_interface == '':
 207             # take first interface by default
 208             connected_sw = self.net.DCNetwork_graph.neighbors(vnf_name)[0]
 209             link_dict = self.net.DCNetwork_graph[vnf_name][connected_sw]
 210             vnf_interface = link_dict[0]['src_port_id']
 211
 212         network_metric['vnf_name'] = vnf_name
 213         network_metric['vnf_interface'] = vnf_interface
 214
 215         for connected_sw in self.net.DCNetwork_graph.neighbors(vnf_name):
 216             link_dict = self.net.DCNetwork_graph[vnf_name][connected_sw]
 217             for link in link_dict:
 218                 if link_dict[link]['src_port_id'] == vnf_interface:
 219                     # found the right link and connected switch
 220                     network_metric['mon_port'] = link_dict[link]['dst_port_nr']
 221                     break
 222
 223         if 'mon_port' not in network_metric:
 224             logging.exception("vnf interface {0}:{1} not found!".format(vnf_name,vnf_interface))
 225             return "vnf interface {0}:{1} not found!".format(vnf_name,vnf_interface)
 226
 227         try:
 228             # default port direction to monitor
 229             if metric is None:
 230                 metric = 'tx_packets'
 231
 232             vnf_switch = self.net.DCNetwork_graph.neighbors(str(vnf_name))
 233
 234             if len(vnf_switch) > 1:
 235                 logging.info("vnf: {0} has multiple ports".format(vnf_name))
 236                 return
 237             elif len(vnf_switch) == 0:
 238                 logging.info("vnf: {0} is not connected".format(vnf_name))
 239                 return
 240             else:
 241                 vnf_switch = vnf_switch[0]
 242             next_node = self.net.getNodeByName(vnf_switch)
 243
 244             if not isinstance(next_node, OVSSwitch):
 245                 logging.info("vnf: {0} is not connected to switch".format(vnf_name))
 246                 return
 247
 248             network_metric['previous_measurement'] = 0
 249             network_metric['previous_monitor_time'] = 0
 250
 251
 252             network_metric['switch_dpid'] = int(str(next_node.dpid), 16)
 253             network_metric['metric_key'] = metric
 254
 255             self.monitor_lock.acquire()
 256             self.network_metrics.append(network_metric)
 257             self.monitor_lock.release()
 258
 259
 260             logging.info('Started monitoring: {2} on {0}:{1}'.format(vnf_name, vnf_interface, metric))
 261             return 'Started monitoring: {2} on {0}:{1}'.format(vnf_name, vnf_interface, metric)
 262
 263         except Exception as ex:
 264             logging.exception("setup_metric error.")
 265             return ex.message
 266
 267     def stop_metric(self, vnf_name, vnf_interface=None, metric=None):
 268
 269         # check if port is specified (vnf:port)
 270         if vnf_interface is None and metric is not None:
 271             # take first interface by default
 272             connected_sw = self.net.DCNetwork_graph.neighbors(vnf_name)[0]
 273             link_dict = self.net.DCNetwork_graph[vnf_name][connected_sw]
 274             vnf_interface = link_dict[0]['src_port_id']
 275
 276         for metric_dict in deepcopy(self.network_metrics):
 277             if metric_dict['vnf_name'] == vnf_name and metric_dict['vnf_interface'] == vnf_interface \
 278                     and metric_dict['metric_key'] == metric:
 279
 280                 self.monitor_lock.acquire()
 281
 282                 self.network_metrics.remove(metric_dict)
 283
 284                 # set values to NaN, prometheus api currently does not support removal of metrics
 285                 #self.prom_metrics[metric_dict['metric_key']].labels(vnf_name, vnf_interface).set(float('nan'))
 286                 self.prom_metrics[metric_dict['metric_key']]. \
 287                     labels(vnf_name=vnf_name, vnf_interface=vnf_interface, flow_id=None). \
 288                     set(float('nan'))
 289
 290                 # this removes the complete metric, all labels...
 291                 # 1 single monitor job for all metrics of the SDN controller
 292                 # we can only  remove from the pushgateway grouping keys(labels) which we have defined for the add_to_pushgateway
 293                 # we can not specify labels from the metrics to be removed
 294                 # if we need to remove the metrics seperatelty, we need to give them a separate grouping key, and probably a diffferent registry also
 295                 delete_from_gateway(self.pushgateway, job='sonemu-SDNcontroller')
 296
 297                 self.monitor_lock.release()
 298
 299                 logging.info('Stopped monitoring: {2} on {0}:{1}'.format(vnf_name, vnf_interface, metric))
 300                 return 'Stopped monitoring: {2} on {0}:{1}'.format(vnf_name, vnf_interface, metric)
 301
 302             # delete everything from this vnf
 303             elif metric_dict['vnf_name'] == vnf_name and vnf_interface is None and metric is None:
 304                 self.monitor_lock.acquire()
 305                 self.network_metrics.remove(metric_dict)
 306                 logging.info('remove metric from monitor: vnf_name:{0} vnf_interface:{1} mon_port:{2}'.format(metric_dict['vnf_name'], metric_dict['vnf_interface'], metric_dict['mon_port']))
 307
 308                 delete_from_gateway(self.pushgateway, job='sonemu-SDNcontroller')
 309                 self.monitor_lock.release()
 310                 continue
 311
 312         if vnf_interface is None and metric is None:
 313             logging.info('Stopped monitoring vnf: {0}'.format(vnf_name))
 314             return 'Stopped monitoring: {0}'.format(vnf_name)
 315         else:
 316             return 'Error stopping monitoring metric: {0} on {1}:{2}'.format(metric, vnf_name, vnf_interface)
 317
 318
 319 # get all metrics defined in the list and export it to Prometheus
 320     def get_flow_metrics(self):
 321         while self.start_monitoring:
 322
 323             self.monitor_flow_lock.acquire()
 324
 325             for flow_dict in self.flow_metrics:
 326                 data = {}
 327
 328                 data['cookie'] = flow_dict['cookie']
 329                 data['cookie_mask'] = COOKIE_MASK
 330
 331                 if 'tx' in flow_dict['metric_key']:
 332                     data['match'] = {'in_port':flow_dict['mon_port']}
 333                 elif 'rx' in flow_dict['metric_key']:
 334                     data['out_port'] = flow_dict['mon_port']
 335
 336
 337                 # query Ryu
 338                 ret = self.net.ryu_REST('stats/flow', dpid=flow_dict['switch_dpid'], data=data)
 339                 if isinstance(ret, dict):
 340                     flow_stat_dict = ret
 341                 elif isinstance(ret, basestring):
 342                     flow_stat_dict = ast.literal_eval(ret.rstrip())
 343                 else:
 344                     flow_stat_dict = None
 345
 346                 logging.debug('received flow stat:{0} '.format(flow_stat_dict))
 347
 348                 self.set_flow_metric(flow_dict, flow_stat_dict)
 349
 350
 351             try:
 352                 if len(self.flow_metrics) > 0:
 353                     pushadd_to_gateway(self.pushgateway, job='sonemu-SDNcontroller', registry=self.registry)
 354             except Exception, e:
 355                 logging.warning("Pushgateway not reachable: {0} {1}".format(Exception, e))
 356
 357             self.monitor_flow_lock.release()
 358             time.sleep(1)
 359
 360     def get_network_metrics(self):
 361         while self.start_monitoring:
 362
 363             self.monitor_lock.acquire()
 364
 365             # group metrics by dpid to optimize the rest api calls
 366             dpid_list = [metric_dict['switch_dpid'] for metric_dict in self.network_metrics]
 367             dpid_set = set(dpid_list)
 368
 369             for dpid in dpid_set:
 370
 371                 # query Ryu
 372                 ret = self.net.ryu_REST('stats/port', dpid=dpid)
 373                 if isinstance(ret, dict):
 374                     port_stat_dict = ret
 375                 elif isinstance(ret, basestring):
 376                     port_stat_dict = ast.literal_eval(ret.rstrip())
 377                 else:
 378                     port_stat_dict = None
 379
 380                 metric_list = [metric_dict for metric_dict in self.network_metrics
 381                                if int(metric_dict['switch_dpid'])==int(dpid)]
 382
 383                 for metric_dict in metric_list:
 384                     self.set_network_metric(metric_dict, port_stat_dict)
 385
 386             try:
 387                 if len(self.network_metrics) > 0:
 388                     pushadd_to_gateway(self.pushgateway, job='sonemu-SDNcontroller', registry=self.registry)
 389             except Exception, e:
 390                 logging.warning("Pushgateway not reachable: {0} {1}".format(Exception, e))
 391
 392             self.monitor_lock.release()
 393             time.sleep(1)
 394
 395     # add metric to the list to export to Prometheus, parse the Ryu port-stats reply
 396     def set_network_metric(self, metric_dict, port_stat_dict):
 397         # vnf tx is the datacenter switch rx and vice-versa
 398         metric_key = self.switch_tx_rx(metric_dict['metric_key'])
 399         switch_dpid = metric_dict['switch_dpid']
 400         vnf_name = metric_dict['vnf_name']
 401         vnf_interface = metric_dict['vnf_interface']
 402         previous_measurement = metric_dict['previous_measurement']
 403         previous_monitor_time = metric_dict['previous_monitor_time']
 404         mon_port = metric_dict['mon_port']
 405         for port_stat in port_stat_dict[str(switch_dpid)]:
 406             # ovs output also gives back 'LOCAL' port
 407             if port_stat['port_no'] == 'LOCAL':
 408                 continue
 409             if int(port_stat['port_no']) == int(mon_port):
 410                 port_uptime = port_stat['duration_sec'] + port_stat['duration_nsec'] * 10 ** (-9)
 411                 this_measurement = int(port_stat[metric_key])
 412
 413                 # set prometheus metric
 414                 self.prom_metrics[metric_dict['metric_key']].\
 415                     labels(vnf_name=vnf_name, vnf_interface=vnf_interface, flow_id=None).\
 416                     set(this_measurement)
 417
 418                 # also the rate is calculated here, but not used for now
 419                 # (rate can be easily queried from prometheus also)
 420                 if previous_monitor_time <= 0 or previous_monitor_time >= port_uptime:
 421                     metric_dict['previous_measurement'] = int(port_stat[metric_key])
 422                     metric_dict['previous_monitor_time'] = port_uptime
 423                     # do first measurement
 424                     #time.sleep(1)
 425                     #self.monitor_lock.release()
 426                     # rate cannot be calculated yet (need a first measurement)
 427                     metric_rate = None
 428
 429                 else:
 430                     time_delta = (port_uptime - metric_dict['previous_monitor_time'])
 431                     #metric_rate = (this_measurement - metric_dict['previous_measurement']) / float(time_delta)
 432
 433                 metric_dict['previous_measurement'] = this_measurement
 434                 metric_dict['previous_monitor_time'] = port_uptime
 435                 return
 436
 437         logging.exception('metric {0} not found on {1}:{2}'.format(metric_key, vnf_name, vnf_interface))
 438         logging.exception('monport:{0}, dpid:{1}'.format(mon_port, switch_dpid))
 439         logging.exception('monitored network_metrics:{0}'.format(self.network_metrics))
 440         logging.exception('port dict:{0}'.format(port_stat_dict))
 441         return 'metric {0} not found on {1}:{2}'.format(metric_key, vnf_name, vnf_interface)
 442
 443     def set_flow_metric(self, metric_dict, flow_stat_dict):
 444         # vnf tx is the datacenter switch rx and vice-versa
 445         metric_key = metric_dict['metric_key']
 446         switch_dpid = metric_dict['switch_dpid']
 447         vnf_name = metric_dict['vnf_name']
 448         vnf_interface = metric_dict['vnf_interface']
 449         previous_measurement = metric_dict['previous_measurement']
 450         previous_monitor_time = metric_dict['previous_monitor_time']
 451         cookie = metric_dict['cookie']
 452
 453         counter = 0
 454         for flow_stat in flow_stat_dict[str(switch_dpid)]:
 455             if 'bytes' in metric_key:
 456                 counter += flow_stat['byte_count']
 457             elif 'packet' in metric_key:
 458                 counter += flow_stat['packet_count']
 459
 460         # flow_uptime disabled for now (can give error)
 461         #flow_stat = flow_stat_dict[str(switch_dpid)][0]
 462         #flow_uptime = flow_stat['duration_sec'] + flow_stat['duration_nsec'] * 10 ** (-9)
 463
 464         self.prom_metrics[metric_dict['metric_key']]. \
 465             labels(vnf_name=vnf_name, vnf_interface=vnf_interface, flow_id=cookie). \
 466             set(counter)
 467
 468     def start_Prometheus(self, port=9090):
 469         # prometheus.yml configuration file is located in the same directory as this file
 470         cmd = ["docker",
 471                "run",
 472                "--rm",
 473                "-p", "{0}:9090".format(port),
 474                "-v", "{0}/prometheus.yml:/etc/prometheus/prometheus.yml".format(os.path.dirname(os.path.abspath(__file__))),
 475                "-v", "{0}/profile.rules:/etc/prometheus/profile.rules".format(os.path.dirname(os.path.abspath(__file__))),
 476                "--name", "prometheus",
 477                "prom/prometheus"
 478                ]
 479         logging.info('Start Prometheus container {0}'.format(cmd))
 480         return Popen(cmd)
 481
 482     def start_PushGateway(self, port=PUSHGATEWAY_PORT):
 483         cmd = ["docker",
 484                "run",
 485                "-d",
 486                "-p", "{0}:9091".format(port),
 487                "--name", "pushgateway",
 488                "--label", 'com.containernet=""',
 489                "prom/pushgateway"
 490                ]
 491
 492         logging.info('Start Prometheus Push Gateway container {0}'.format(cmd))
 493         return Popen(cmd)
 494
 495     def start_cAdvisor(self, port=CADVISOR_PORT):
 496         cmd = ["docker",
 497                "run",
 498                "--rm",
 499                "--volume=/:/rootfs:ro",
 500                "--volume=/var/run:/var/run:rw",
 501                "--volume=/sys:/sys:ro",
 502                "--volume=/var/lib/docker/:/var/lib/docker:ro",
 503                "--publish={0}:8080".format(port),
 504                "--name=cadvisor",
 505                "--label",'com.containernet=""',
 506                "--detach=true",
 507                "google/cadvisor:latest",
 508                #"--storage_duration=1m0s",
 509                #"--allow_dynamic_housekeeping=true",
 510                #"--housekeeping_interval=1s",
 511                ]
 512         logging.info('Start cAdvisor container {0}'.format(cmd))
 513         return Popen(cmd)
 514
 515     def stop(self):
 516         # stop the monitoring thread
 517         self.start_monitoring = False
 518         self.monitor_thread.join()
 519         self.monitor_flow_thread.join()
 520
 521         # these containers are used for monitoring but are started now outside of son-emu
 522
 523         if self.pushgateway_process is not None:
 524             logging.info('stopping pushgateway container')
 525             self._stop_container('pushgateway')
 526
 527         if self.cadvisor_process is not None:
 528             logging.info('stopping cadvisor container')
 529             self._stop_container('cadvisor')
 530
 531     def switch_tx_rx(self,metric=''):
 532         # when monitoring vnfs, the tx of the datacenter switch is actually the rx of the vnf
 533         # so we need to change the metric name to be consistent with the vnf rx or tx
 534         if 'tx' in metric:
 535             metric = metric.replace('tx','rx')
 536         elif 'rx' in metric:
 537             metric = metric.replace('rx','tx')
 538
 539         return metric
 540
 541     def _stop_container(self, name):
 542
 543         #container = self.dockercli.containers.get(name)
 544         #container.stop()
 545         #container.remove(force=True)
 546
 547         # the only robust way to stop these containers is via Popen, it seems
 548         time.sleep(1)
 549         cmd = ['docker', 'rm', '-f', name]
 550         Popen(cmd)
 551
 552
 553     def update_skewmon(self, vnf_name, resource_name, action):
 554
 555         ret = ''
 556
 557         config_file_path = '/tmp/skewmon.cfg'
 558         configfile = open(config_file_path, 'a+')
 559         try:
 560             config = json.load(configfile)
 561         except:
 562             #not a valid json file or empty
 563             config = {}
 564
 565         #initialize config file
 566         if len(self.skewmon_metrics) == 0:
 567             config = {}
 568         json.dump(config, configfile)
 569         configfile.close()
 570
 571         docker_name = 'mn.' + vnf_name
 572         vnf_container = self.dockercli.containers.get(docker_name)
 573         key = resource_name + '_' + vnf_container.short_id
 574         vnf_id = vnf_container.id
 575
 576         if action == 'start':
 577             # add a new vnf to monitor
 578             config[key] = dict(VNF_NAME=vnf_name,
 579                                 VNF_ID=vnf_id,
 580                                 VNF_METRIC=resource_name)
 581             ret = 'adding to skewness monitor: {0} {1} '.format(vnf_name, resource_name)
 582             logging.info(ret)
 583         elif action == 'stop':
 584             # remove vnf to monitor
 585             config.pop(key)
 586             ret = 'removing from skewness monitor: {0} {1} '.format(vnf_name, resource_name)
 587             logging.info(ret)
 588
 589         self.skewmon_metrics = config
 590         configfile = open(config_file_path, 'w')
 591         json.dump(config, configfile)
 592         configfile.close()
 593
 594         try:
 595             skewmon_container = self.dockercli.containers.get('skewmon')
 596
 597             # remove container if config is empty
 598             if len(config) == 0:
 599                 ret += 'stopping skewness monitor'
 600                 logging.info('stopping skewness monitor')
 601                 skewmon_container.remove(force=True)
 602
 603         except docker.errors.NotFound:
 604             # start container if not running
 605             ret += 'starting skewness monitor'
 606             logging.info('starting skewness monitor')
 607             volumes = {'/sys/fs/cgroup':{'bind':'/sys/fs/cgroup', 'mode':'ro'},
 608                        '/tmp/skewmon.cfg':{'bind':'/config.txt', 'mode':'ro'}}
 609             self.dockercli.containers.run('skewmon',
 610                                           detach=True,
 611                                           volumes=volumes,
 612                                           labels=['com.containernet'],
 613                                           name='skewmon'
 614                                           )
 615             # Wait a while for containers to be completely started
 616             started = False
 617             wait_time = 0
 618             while not started:
 619                 list1 = self.dockercli.containers.list(filters={'status': 'running', 'name': 'prometheus'})
 620                 if len(list1) >= 1:
 621                     time.sleep(1)
 622                     started = True
 623                 if wait_time > 5:
 624                     return 'skewmon not started'
 625                 time.sleep(1)
 626                 wait_time += 1
 627         return ret
 628
 629     def term(self, vnf_list=[]):
 630         """
 631         Start a terminal window for the specified VNFs
 632         (start a terminal for all VNFs if vnf_list is empty)
 633         :param vnf_list:
 634         :return:
 635         """
 636
 637
 638         if vnf_list is None:
 639             vnf_list = []
 640         if not isinstance(vnf_list, list):
 641             vnf_list = str(vnf_list).split(',')
 642             vnf_list = map(str.strip, vnf_list)
 643         logging.info('vnf_list: {}'.format(vnf_list))
 644
 645         return self.start_xterm(vnf_list)
 646
 647
 648     # start an xterm for the specfified vnfs
 649     def start_xterm(self, vnf_names):
 650         # start xterm for all vnfs
 651         for vnf_name in vnf_names:
 652             terminal_cmd = "docker exec -it mn.{0} /bin/bash".format(vnf_name)
 653
 654             cmd = ['xterm', '-xrm', 'XTerm*selectToClipboard: true', '-xrm', 'XTerm.vt100.allowTitleOps: false',
 655                    '-T', vnf_name,
 656                    '-e', terminal_cmd]
 657             Popen(cmd)
 658
 659         ret = 'xterms started for {0}'.format(vnf_names)
 660         if len(vnf_names) == 0:
 661             ret = 'vnf list is empty, no xterms started'
 662         return ret
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672