fix label issues with prometheus api+clean stop of metric export

diff --git a/src/emuvim/dcemulator/monitoring.py b/src/emuvim/dcemulator/monitoring.py
index 9cf2a3b..28c3df6 100755 (executable)
--- a/src/emuvim/dcemulator/monitoring.py
+++ b/src/emuvim/dcemulator/monitoring.py
@@ -36,7 +36,7 @@ from prometheus_client import start_http_server, Summary, Histogram, Gauge, Coun
 import threading\r
 from subprocess import Popen\r
 import os\r
-\r
+import docker\r
 \r
 logging.basicConfig(level=logging.INFO)\r
 \r
@@ -53,6 +53,7 @@ COOKIE_MASK = 0xffffffff
 class DCNetworkMonitor():\r
     def __init__(self, net):\r
         self.net = net\r
+        self.dockercli = docker.from_env()\r
 \r
         # pushgateway address\r
         self.pushgateway = 'localhost:{0}'.format(PUSHGATEWAY_PORT)\r
@@ -178,9 +179,10 @@ class DCNetworkMonitor():
 \r
                 self.flow_metrics.remove(flow_dict)\r
 \r
-                for collector in self.registry._collectors:\r
-                    if (vnf_name, vnf_interface, cookie) in collector._metrics:\r
-                        collector.remove(vnf_name, vnf_interface, cookie)\r
+                # set metric to NaN\r
+                self.prom_metrics[flow_dict['metric_key']]. \\r
+                    labels(vnf_name=vnf_name, vnf_interface=vnf_interface, flow_id=cookie). \\r
+                    set(float('nan'))\r
 \r
                 delete_from_gateway(self.pushgateway, job='sonemu-SDNcontroller')\r
 \r
@@ -276,29 +278,11 @@ class DCNetworkMonitor():
 \r
                 self.network_metrics.remove(metric_dict)\r
 \r
-                #this removes the complete metric, all labels...\r
-                #REGISTRY.unregister(self.prom_metrics[metric_dict['metric_key']])\r
-                #self.registry.unregister(self.prom_metrics[metric_dict['metric_key']])\r
-\r
-                for collector in self.registry._collectors :\r
-\r
-                    """\r
-                    INFO:root:name:sonemu_rx_count_packets\r
-                    labels:('vnf_name', 'vnf_interface')\r
-                    metrics:{(u'tsrc', u'output'): < prometheus_client.core.Gauge\r
-                    object\r
-                    at\r
-                    0x7f353447fd10 >}\r
-                    """\r
-                    logging.info('{0}'.format(collector._metrics.values()))\r
-\r
-                    if (vnf_name, vnf_interface, 'None') in collector._metrics:\r
-                        logging.info('2 name:{0} labels:{1} metrics:{2}'.format(collector._name, collector._labelnames,\r
-                                                                              collector._metrics))\r
-                        collector.remove(vnf_name, vnf_interface, 'None')\r
-\r
                 # set values to NaN, prometheus api currently does not support removal of metrics\r
                 #self.prom_metrics[metric_dict['metric_key']].labels(vnf_name, vnf_interface).set(float('nan'))\r
+                self.prom_metrics[metric_dict['metric_key']]. \\r
+                    labels(vnf_name=vnf_name, vnf_interface=vnf_interface, flow_id=None). \\r
+                    set(float('nan'))\r
 \r
                 # this removes the complete metric, all labels...\r
                 # 1 single monitor job for all metrics of the SDN controller\r
@@ -363,6 +347,13 @@ class DCNetworkMonitor():
 \r
                 self.set_flow_metric(flow_dict, flow_stat_dict)\r
 \r
+\r
+            try:\r
+                if len(self.flow_metrics) > 0:\r
+                    pushadd_to_gateway(self.pushgateway, job='sonemu-SDNcontroller', registry=self.registry)\r
+            except Exception, e:\r
+                logging.warning("Pushgateway not reachable: {0} {1}".format(Exception, e))\r
+\r
             self.monitor_flow_lock.release()\r
             time.sleep(1)\r
 \r
@@ -392,6 +383,12 @@ class DCNetworkMonitor():
                 for metric_dict in metric_list:\r
                     self.set_network_metric(metric_dict, port_stat_dict)\r
 \r
+            try:\r
+                if len(self.network_metrics) > 0:\r
+                    pushadd_to_gateway(self.pushgateway, job='sonemu-SDNcontroller', registry=self.registry)\r
+            except Exception, e:\r
+                logging.warning("Pushgateway not reachable: {0} {1}".format(Exception, e))\r
+\r
             self.monitor_lock.release()\r
             time.sleep(1)\r
 \r
@@ -413,12 +410,9 @@ class DCNetworkMonitor():
 \r
                 # set prometheus metric\r
                 self.prom_metrics[metric_dict['metric_key']].\\r
-                    labels({'vnf_name': vnf_name, 'vnf_interface': vnf_interface, 'flow_id': None}).\\r
+                    labels(vnf_name=vnf_name, vnf_interface=vnf_interface, flow_id=None).\\r
                     set(this_measurement)\r
 \r
-                # 1 single monitor job for all metrics of the SDN controller\r
-                pushadd_to_gateway(self.pushgateway, job='sonemu-SDNcontroller', registry=self.registry)\r
-\r
                 # also the rate is calculated here, but not used for now\r
                 # (rate can be easily queried from prometheus also)\r
                 if previous_monitor_time <= 0 or previous_monitor_time >= port_uptime:\r
@@ -465,13 +459,8 @@ class DCNetworkMonitor():
         #flow_uptime = flow_stat['duration_sec'] + flow_stat['duration_nsec'] * 10 ** (-9)\r
 \r
         self.prom_metrics[metric_dict['metric_key']]. \\r
-            labels({'vnf_name': vnf_name, 'vnf_interface': vnf_interface, 'flow_id': cookie}). \\r
+            labels(vnf_name=vnf_name, vnf_interface=vnf_interface, flow_id=cookie). \\r
             set(counter)\r
-        try:\r
-            pushadd_to_gateway(self.pushgateway, job='sonemu-SDNcontroller', registry=self.registry)\r
-        except Exception, e:\r
-            logging.warning("Pushgateway not reachable: {0} {1}".format(Exception, e))\r
-\r
 \r
     def start_Prometheus(self, port=9090):\r
         # prometheus.yml configuration file is located in the same directory as this file\r
@@ -523,23 +512,13 @@ class DCNetworkMonitor():
         self.monitor_flow_thread.join()\r
 \r
         # these containers are used for monitoring but are started now outside of son-emu\r
-        '''\r
-        if self.prometheus_process is not None:\r
-            logging.info('stopping prometheus container')\r
-            self.prometheus_process.terminate()\r
-            self.prometheus_process.kill()\r
-            self._stop_container('prometheus')\r
-        '''\r
+\r
         if self.pushgateway_process is not None:\r
             logging.info('stopping pushgateway container')\r
-            #self.pushgateway_process.terminate()\r
-            #self.pushgateway_process.kill()\r
             self._stop_container('pushgateway')\r
 \r
         if self.cadvisor_process is not None:\r
             logging.info('stopping cadvisor container')\r
-            #self.cadvisor_process.terminate()\r
-            #self.cadvisor_process.kill()\r
             self._stop_container('cadvisor')\r
 \r
     def switch_tx_rx(self,metric=''):\r
@@ -554,9 +533,7 @@ class DCNetworkMonitor():
 \r
     def _stop_container(self, name):\r
 \r
-        cmd = ["docker",\r
-               "rm",\r
-               "-f",\r
-               name]\r
-        Popen(cmd).wait()\r
+        container = self.dockercli.containers.get(name)\r
+        container.remove(force=True)\r
+\r
 \r