fix bug in monitoring (deleting flow metrics sometimes crashes)

[osm/vim-emu.git] / src / emuvim / dcemulator / monitoring.py

diff --git a/src/emuvim/dcemulator/monitoring.py b/src/emuvim/dcemulator/monitoring.py
index 2e1e041..9cf2a3b 100755 (executable)
--- a/src/emuvim/dcemulator/monitoring.py
+++ b/src/emuvim/dcemulator/monitoring.py
@@ -45,7 +45,10 @@ class to read openflow stats from the Ryu controller of the DCNetwork
 """\r
 \r
 PUSHGATEWAY_PORT = 9091\r
-CADVISOR_PORT = 8080\r
+# we cannot use port 8080 because ryu-ofrest api  is already using that one\r
+CADVISOR_PORT = 8081\r
+\r
+COOKIE_MASK = 0xffffffff\r
 \r
 class DCNetworkMonitor():\r
     def __init__(self, net):\r
@@ -96,8 +99,8 @@ class DCNetworkMonitor():
 \r
         # helper tools\r
         # cAdvisor, Prometheus pushgateway are started as external container, to gather monitoring metric in son-emu\r
-        self.start_PushGateway()\r
-        self.start_cAdvisor()\r
+        self.pushgateway_process = self.start_PushGateway()\r
+        self.cadvisor_process = self.start_cAdvisor()\r
 \r
 \r
     # first set some parameters, before measurement can start\r
@@ -245,7 +248,6 @@ class DCNetworkMonitor():
             network_metric['metric_key'] = metric\r
 \r
             self.monitor_lock.acquire()\r
-\r
             self.network_metrics.append(network_metric)\r
             self.monitor_lock.release()\r
 \r
@@ -330,9 +332,6 @@ class DCNetworkMonitor():
         return 'Error stopping monitoring metric: {0} on {1}:{2}'.format(metric, vnf_name, vnf_interface)\r
 \r
 \r
-\r
-\r
-\r
 # get all metrics defined in the list and export it to Prometheus\r
     def get_flow_metrics(self):\r
         while self.start_monitoring:\r
@@ -343,7 +342,7 @@ class DCNetworkMonitor():
                 data = {}\r
 \r
                 data['cookie'] = flow_dict['cookie']\r
-                data['cookie_mask'] = flow_dict['cookie']\r
+                data['cookie_mask'] = COOKIE_MASK\r
 \r
                 if 'tx' in flow_dict['metric_key']:\r
                     data['match'] = {'in_port':flow_dict['mon_port']}\r
@@ -380,7 +379,12 @@ class DCNetworkMonitor():
 \r
                 # query Ryu\r
                 ret = self.net.ryu_REST('stats/port', dpid=dpid)\r
-                port_stat_dict = ast.literal_eval(ret)\r
+                if isinstance(ret, dict):\r
+                    port_stat_dict = ret\r
+                elif isinstance(ret, basestring):\r
+                    port_stat_dict = ast.literal_eval(ret.rstrip())\r
+                else:\r
+                    port_stat_dict = None\r
 \r
                 metric_list = [metric_dict for metric_dict in self.network_metrics\r
                                if int(metric_dict['switch_dpid'])==int(dpid)]\r
@@ -421,11 +425,10 @@ class DCNetworkMonitor():
                     metric_dict['previous_measurement'] = int(port_stat[metric_key])\r
                     metric_dict['previous_monitor_time'] = port_uptime\r
                     # do first measurement\r
-                    time.sleep(1)\r
-                    self.monitor_lock.release()\r
-\r
-                    metric_rate = self.get_network_metrics()\r
-                    return metric_rate\r
+                    #time.sleep(1)\r
+                    #self.monitor_lock.release()\r
+                    # rate cannot be calculated yet (need a first measurement)\r
+                    metric_rate = None\r
 \r
                 else:\r
                     time_delta = (port_uptime - metric_dict['previous_monitor_time'])\r
@@ -433,9 +436,11 @@ class DCNetworkMonitor():
 \r
                 metric_dict['previous_measurement'] = this_measurement\r
                 metric_dict['previous_monitor_time'] = port_uptime\r
-                return metric_rate\r
+                return\r
 \r
         logging.exception('metric {0} not found on {1}:{2}'.format(metric_key, vnf_name, vnf_interface))\r
+        logging.exception('monport:{0}, dpid:{1}'.format(mon_port, switch_dpid))\r
+        logging.exception('port dict:{0}'.format(port_stat_dict))\r
         return 'metric {0} not found on {1}:{2}'.format(metric_key, vnf_name, vnf_interface)\r
 \r
     def set_flow_metric(self, metric_dict, flow_stat_dict):\r
@@ -455,8 +460,9 @@ class DCNetworkMonitor():
             elif 'packet' in metric_key:\r
                 counter += flow_stat['packet_count']\r
 \r
-        flow_stat = flow_stat_dict[str(switch_dpid)][0]\r
-        flow_uptime = flow_stat['duration_sec'] + flow_stat['duration_nsec'] * 10 ** (-9)\r
+        # flow_uptime disabled for now (can give error)\r
+        #flow_stat = flow_stat_dict[str(switch_dpid)][0]\r
+        #flow_uptime = flow_stat['duration_sec'] + flow_stat['duration_nsec'] * 10 ** (-9)\r
 \r
         self.prom_metrics[metric_dict['metric_key']]. \\r
             labels({'vnf_name': vnf_name, 'vnf_interface': vnf_interface, 'flow_id': cookie}). \\r
@@ -467,7 +473,7 @@ class DCNetworkMonitor():
             logging.warning("Pushgateway not reachable: {0} {1}".format(Exception, e))\r
 \r
 \r
-    def start_Prometheus(self, port=CADVISOR_PORT):\r
+    def start_Prometheus(self, port=9090):\r
         # prometheus.yml configuration file is located in the same directory as this file\r
         cmd = ["docker",\r
                "run",\r
@@ -487,13 +493,14 @@ class DCNetworkMonitor():
                "-d",\r
                "-p", "{0}:9091".format(port),\r
                "--name", "pushgateway",\r
+               "--label", 'com.containernet=""',\r
                "prom/pushgateway"\r
                ]\r
 \r
         logging.info('Start Prometheus Push Gateway container {0}'.format(cmd))\r
         return Popen(cmd)\r
 \r
-    def start_cAdvisor(self, port=8080):\r
+    def start_cAdvisor(self, port=CADVISOR_PORT):\r
         cmd = ["docker",\r
                "run",\r
                "--rm",\r
@@ -503,6 +510,7 @@ class DCNetworkMonitor():
                "--volume=/var/lib/docker/:/var/lib/docker:ro",\r
                "--publish={0}:8080".format(port),\r
                "--name=cadvisor",\r
+               "--label",'com.containernet=""',\r
                "google/cadvisor:latest"\r
                ]\r
         logging.info('Start cAdvisor container {0}'.format(cmd))\r
@@ -521,19 +529,18 @@ class DCNetworkMonitor():
             self.prometheus_process.terminate()\r
             self.prometheus_process.kill()\r
             self._stop_container('prometheus')\r
-\r
+        '''\r
         if self.pushgateway_process is not None:\r
             logging.info('stopping pushgateway container')\r
-            self.pushgateway_process.terminate()\r
-            self.pushgateway_process.kill()\r
+            #self.pushgateway_process.terminate()\r
+            #self.pushgateway_process.kill()\r
             self._stop_container('pushgateway')\r
 \r
         if self.cadvisor_process is not None:\r
             logging.info('stopping cadvisor container')\r
-            self.cadvisor_process.terminate()\r
-            self.cadvisor_process.kill()\r
+            #self.cadvisor_process.terminate()\r
+            #self.cadvisor_process.kill()\r
             self._stop_container('cadvisor')\r
-        '''\r
 \r
     def switch_tx_rx(self,metric=''):\r
         # when monitoring vnfs, the tx of the datacenter switch is actually the rx of the vnf\r
@@ -546,13 +553,10 @@ class DCNetworkMonitor():
         return metric\r
 \r
     def _stop_container(self, name):\r
-        cmd = ["docker",\r
-               "stop",\r
-               name]\r
-        Popen(cmd).wait()\r
 \r
         cmd = ["docker",\r
                "rm",\r
+               "-f",\r
                name]\r
         Popen(cmd).wait()\r
 \r