Merge pull request #137 from stevenvanrossem/master
authorstevenvanrossem <steven.vanrossem@intec.ugent.be>
Sun, 17 Jul 2016 00:21:00 +0000 (02:21 +0200)
committerpeusterm <manuel.peuster@uni-paderborn.de>
Sun, 17 Jul 2016 00:21:00 +0000 (02:21 +0200)
cleanup monitoring REST api

src/emuvim/api/rest/compute.py
src/emuvim/api/rest/monitor.py
src/emuvim/api/rest/rest_api_endpoint.py
src/emuvim/cli/rest/monitor.py
src/emuvim/cli/rest/network.py
src/emuvim/dcemulator/monitoring.py
src/emuvim/examples/son-monitor_default_topo.py [new file with mode: 0755]
utils/docker/Dockerfile
utils/docker/docker-compose_son-emu.yml [new file with mode: 0755]
utils/docker/entrypoint.sh
utils/docker/entrypoint_docker-compose.sh [new file with mode: 0755]

index e412133..9f0516b 100755 (executable)
@@ -78,7 +78,8 @@ class ComputeStart(Resource):
         '''
         nw_list = list()
 
-        if network_str is None or '),(' not in network_str :
+        # TODO make this more robust with regex check
+        if network_str is None :
             return nw_list
 
         networks = network_str[1:-1].split('),(')
index 15ec014..45d9541 100755 (executable)
@@ -53,7 +53,7 @@ class MonitorInterfaceAction(Resource):
     """
     global net
 
-    def put(self, vnf_name, vnf_interface, metric):
+    def put(self, vnf_name, vnf_interface=None, metric='tx_packets'):
         logging.debug("REST CALL: start monitor VNF interface")
         try:
             c = net.monitor_agent.setup_metric(vnf_name, vnf_interface, metric)
@@ -63,7 +63,7 @@ class MonitorInterfaceAction(Resource):
             logging.exception("API error.")
             return ex.message, 500
 
-    def delete(self, vnf_name, vnf_interface, metric):
+    def delete(self, vnf_name, vnf_interface=None, metric='tx_packets'):
         logging.debug("REST CALL: stop monitor VNF interface")
         try:
             c = net.monitor_agent.stop_metric(vnf_name, vnf_interface, metric)
@@ -85,7 +85,7 @@ class MonitorFlowAction(Resource):
     """
     global net
 
-    def put(self, vnf_name, vnf_interface, metric, cookie):
+    def put(self, vnf_name, vnf_interface=None, metric='tx_packets', cookie=0):
         logging.debug("REST CALL: start monitor VNF interface")
         try:
             c = net.monitor_agent.setup_flow(vnf_name, vnf_interface, metric, cookie)
@@ -95,7 +95,7 @@ class MonitorFlowAction(Resource):
             logging.exception("API error.")
             return ex.message, 500
 
-    def delete(self, vnf_name, vnf_interface, metric, cookie):
+    def delete(self, vnf_name, vnf_interface=None, metric='tx_packets', cookie=0):
         logging.debug("REST CALL: stop monitor VNF interface")
         try:
             c = net.monitor_agent.stop_flow(vnf_name, vnf_interface, metric, cookie)
index d6c1185..536ed7a 100755 (executable)
@@ -69,10 +69,14 @@ class RestApiEndpoint(object):
         self.api.add_resource(DatacenterList, "/restapi/datacenter")
         self.api.add_resource(DatacenterStatus, "/restapi/datacenter/<dc_label>")
 
-        self.api.add_resource(NetworkAction, "/restapi/network/<vnf_src_name>/<vnf_dst_name>")
-
-        self.api.add_resource(MonitorInterfaceAction, "/restapi/monitor/<vnf_name>/<vnf_interface>/<metric>")
-        self.api.add_resource(MonitorFlowAction, "/restapi/monitor/<vnf_name>/<vnf_interface>/<metric>/<cookie>")
+        self.api.add_resource(NetworkAction, "/restapi/network/<vnf_src_name>/<vnf_dst_name>",)
+
+        self.api.add_resource(MonitorInterfaceAction,
+                              "/restapi/monitor/<vnf_name>/<metric>",
+                              "/restapi/monitor/<vnf_name>/<vnf_interface>/<metric>")
+        self.api.add_resource(MonitorFlowAction,
+                              "/restapi/flowmon/<vnf_name>/<metric>/<cookie>",
+                              "/restapi/flowmon/<vnf_name>/<vnf_interface>/<metric>/<cookie>")
 
         logging.debug("Created API endpoint %s(%s:%d)" % (self.__class__.__name__, self.ip, self.port))
 
index 7e9b839..858aad7 100755 (executable)
@@ -25,11 +25,10 @@ the Horizon 2020 and 5G-PPP programmes. The authors would like to
 acknowledge the contributions of their colleagues of the SONATA
 partner consortium (www.sonata-nfv.eu).
 """
+
 from requests import get, put, delete
-from tabulate import tabulate
 import pprint
 import argparse
-import json
 from emuvim.cli import prometheus
 
 pp = pprint.PrettyPrinter(indent=4)
index c6e1dcc..86ed92b 100755 (executable)
@@ -26,10 +26,8 @@ acknowledge the contributions of their colleagues of the SONATA
 partner consortium (www.sonata-nfv.eu).
 """
 from requests import get,put, delete
-from tabulate import tabulate
 import pprint
 import argparse
-import json
 
 pp = pprint.PrettyPrinter(indent=4)
 
@@ -114,13 +112,12 @@ parser.add_argument(
     help="vnf name of the destination of the chain")
 parser.add_argument(
     "--weight", "-w", dest="weight",
-    help="weight metric to calculate the path")
+    help="weight edge attribute to calculate the path")
 parser.add_argument(
     "--match", "-m", dest="match",
     help="string holding extra matches for the flow entries")
 parser.add_argument(
     "--bidirectional", "-b", dest="bidirectional",
-    action='store_true',
     help="add/remove the flow entries from src to dst and back")
 parser.add_argument(
     "--cookie", "-c", dest="cookie",
index 89d82ba..78d6ebb 100755 (executable)
@@ -107,7 +107,7 @@ class DCNetworkMonitor():
         # Prometheus pushgateway and DB are started as external contianer, outside of son-emu\r
         #self.pushgateway_process = self.start_PushGateway()\r
         #self.prometheus_process = self.start_Prometheus()\r
-        self.cadvisor_process = self.start_cadvisor()\r
+        #self.cadvisor_process = self.start_cadvisor()\r
 \r
     # first set some parameters, before measurement can start\r
     def setup_flow(self, vnf_name, vnf_interface=None, metric='tx_packets', cookie=0):\r
@@ -167,7 +167,15 @@ class DCNetworkMonitor():
             logging.exception("setup_metric error.")\r
             return ex.message\r
 \r
-    def stop_flow(self, vnf_name, vnf_interface=None, metric=None, cookie=0):\r
+    def stop_flow(self, vnf_name, vnf_interface=None, metric=None, cookie=0,):\r
+\r
+        # check if port is specified (vnf:port)\r
+        if vnf_interface is None and metric is not None:\r
+            # take first interface by default\r
+            connected_sw = self.net.DCNetwork_graph.neighbors(vnf_name)[0]\r
+            link_dict = self.net.DCNetwork_graph[vnf_name][connected_sw]\r
+            vnf_interface = link_dict[0]['src_port_id']\r
+\r
         for flow_dict in self.flow_metrics:\r
             if flow_dict['vnf_name'] == vnf_name and flow_dict['vnf_interface'] == vnf_interface \\r
                     and flow_dict['metric_key'] == metric and flow_dict['cookie'] == cookie:\r
@@ -187,6 +195,8 @@ class DCNetworkMonitor():
                 logging.info('Stopped monitoring flow {3}: {2} on {0}:{1}'.format(vnf_name, vnf_interface, metric, cookie))\r
                 return 'Stopped monitoring flow {3}: {2} on {0}:{1}'.format(vnf_name, vnf_interface, metric, cookie)\r
 \r
+        return 'Error stopping monitoring flow: {0} on {1}:{2}'.format(metric, vnf_name, vnf_interface)\r
+\r
 \r
     # first set some parameters, before measurement can start\r
     def setup_metric(self, vnf_name, vnf_interface=None, metric='tx_packets'):\r
@@ -258,6 +268,13 @@ class DCNetworkMonitor():
 \r
     def stop_metric(self, vnf_name, vnf_interface=None, metric=None):\r
 \r
+        # check if port is specified (vnf:port)\r
+        if vnf_interface is None and metric is not None:\r
+            # take first interface by default\r
+            connected_sw = self.net.DCNetwork_graph.neighbors(vnf_name)[0]\r
+            link_dict = self.net.DCNetwork_graph[vnf_name][connected_sw]\r
+            vnf_interface = link_dict[0]['src_port_id']\r
+\r
         for metric_dict in self.network_metrics:\r
             if metric_dict['vnf_name'] == vnf_name and metric_dict['vnf_interface'] == vnf_interface \\r
                     and metric_dict['metric_key'] == metric:\r
@@ -319,8 +336,13 @@ class DCNetworkMonitor():
                 logging.info('Stopped monitoring vnf: {0}'.format(vnf_name))\r
                 return 'Stopped monitoring: {0}'.format(vnf_name)\r
 \r
+        return 'Error stopping monitoring metric: {0} on {1}:{2}'.format(metric, vnf_name, vnf_interface)\r
+\r
+\r
+\r
 \r
-    # get all metrics defined in the list and export it to Prometheus\r
+\r
+# get all metrics defined in the list and export it to Prometheus\r
     def get_flow_metrics(self):\r
         while self.start_monitoring:\r
 \r
@@ -429,12 +451,20 @@ class DCNetworkMonitor():
         cookie = metric_dict['cookie']\r
 \r
         # TODO aggregate all found flow stats\r
-        flow_stat = flow_stat_dict[str(switch_dpid)][0]\r
-        if 'bytes' in  metric_key:\r
-            counter = flow_stat['byte_count']\r
-        elif 'packet' in metric_key:\r
-            counter = flow_stat['packet_count']\r
+        #flow_stat = flow_stat_dict[str(switch_dpid)][0]\r
+        #if 'bytes' in metric_key:\r
+        #    counter = flow_stat['byte_count']\r
+        #elif 'packet' in metric_key:\r
+        #    counter = flow_stat['packet_count']\r
+\r
+        counter = 0\r
+        for flow_stat in flow_stat_dict[str(switch_dpid)]:\r
+            if 'bytes' in metric_key:\r
+                counter += flow_stat['byte_count']\r
+            elif 'packet' in metric_key:\r
+                counter += flow_stat['packet_count']\r
 \r
+        flow_stat = flow_stat_dict[str(switch_dpid)][0]\r
         flow_uptime = flow_stat['duration_sec'] + flow_stat['duration_nsec'] * 10 ** (-9)\r
 \r
         self.prom_metrics[metric_dict['metric_key']]. \\r
@@ -490,6 +520,7 @@ class DCNetworkMonitor():
         self.monitor_thread.join()\r
         self.monitor_flow_thread.join()\r
 \r
+        # these containers are used for monitoring but are started now outside of son-emu\r
         '''\r
         if self.prometheus_process is not None:\r
             logging.info('stopping prometheus container')\r
@@ -502,13 +533,13 @@ class DCNetworkMonitor():
             self.pushgateway_process.terminate()\r
             self.pushgateway_process.kill()\r
             self._stop_container('pushgateway')\r
-        '''\r
 \r
         if self.cadvisor_process is not None:\r
             logging.info('stopping cadvisor container')\r
             self.cadvisor_process.terminate()\r
             self.cadvisor_process.kill()\r
             self._stop_container('cadvisor')\r
+        '''\r
 \r
     def switch_tx_rx(self,metric=''):\r
         # when monitoring vnfs, the tx of the datacenter switch is actually the rx of the vnf\r
diff --git a/src/emuvim/examples/son-monitor_default_topo.py b/src/emuvim/examples/son-monitor_default_topo.py
new file mode 100755 (executable)
index 0000000..29893ae
--- /dev/null
@@ -0,0 +1,100 @@
+"""
+Copyright (c) 2015 SONATA-NFV
+ALL RIGHTS RESERVED.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Neither the name of the SONATA-NFV [, ANY ADDITIONAL AFFILIATION]
+nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written
+permission.
+
+This work has been performed in the framework of the SONATA project,
+funded by the European Commission under Grant number 671517 through
+the Horizon 2020 and 5G-PPP programmes. The authors would like to
+acknowledge the contributions of their colleagues of the SONATA
+partner consortium (www.sonata-nfv.eu).
+"""
+"""
+A simple topology with two PoPs for the y1 demo story board.
+
+        (dc1) <<-->> s1 <<-->> (dc2)
+"""
+
+import logging
+from mininet.log import setLogLevel
+from emuvim.dcemulator.net import DCNetwork
+from emuvim.api.rest.rest_api_endpoint import RestApiEndpoint
+from emuvim.api.sonata import SonataDummyGatekeeperEndpoint
+from mininet.node import RemoteController
+import signal
+import sys
+
+
+logging.basicConfig(level=logging.INFO)
+
+net = None
+cli = None
+
+
+def create_topology1():
+    # create topology
+    net = DCNetwork(controller=RemoteController, monitor=False, enable_learning = False)
+    dc1 = net.addDatacenter("dc1")
+    dc2 = net.addDatacenter("dc2")
+    s1 = net.addSwitch("s1")
+    net.addLink(dc1, s1, delay="10ms")
+    net.addLink(dc2, s1, delay="20ms")
+
+    # add the command line interface endpoint to each DC (REST API)
+    rapi1 = RestApiEndpoint("0.0.0.0", 5001)
+    rapi1.connectDatacenter(dc1)
+    rapi1.connectDatacenter(dc2)
+    # run API endpoint server (in another thread, don't block)
+    rapi1.start()
+
+    # add the SONATA dummy gatekeeper to each DC
+    sdkg1 = SonataDummyGatekeeperEndpoint("0.0.0.0", 5000)
+    sdkg1.connectDatacenter(dc1)
+    sdkg1.connectDatacenter(dc2)
+    # run the dummy gatekeeper (in another thread, don't block)
+    sdkg1.start()
+
+    # start the emulation platform
+    net.start()
+    #cli = net.CLI()
+    #net.stop()
+
+def exit_gracefully(signum, frame):
+    """
+    7. At shutdown, we should receive th SIGTERM signal here and shutdown gracefully
+    """
+    # TODO: investigate why this is not called by the sigterm handler
+
+    global net
+    global cli
+
+    logging.info('Signal handler called with signal {0}'.format(signum))
+    net.stop()
+
+    sys.exit()
+
+def main():
+    setLogLevel('info')  # set Mininet loglevel
+    # add the SIGTERM handler (eg. received when son-emu docker container stops)
+    signal.signal(signal.SIGTERM, exit_gracefully)
+    create_topology1()
+
+
+if __name__ == '__main__':
+    main()
index c40af0a..31da835 100755 (executable)
@@ -34,17 +34,19 @@ COPY . /son-emu/
 
 RUN apt-get clean
 
-RUN cd /son-emu/ansible \
-    && ansible-playbook install.yml \
-    && cd /son-emu \
-    # we need to reset the __pycache__ for correct test discovery
-    && rm -rf src/emuvim/test/__pycache__ \
-    && rm -rf src/emuvim/test/unittests/__pycache__ \
-    && rm -rf src/emuvim/test/integrationtests/__pycache__ \
-    && python setup.py install \
-    && echo 'Done'
+WORKDIR /son-emu/ansible
+RUN ansible-playbook install.yml
+
+WORKDIR /son-emu
+# we need to reset the __pycache__ for correct test discovery
+RUN rm -rf src/emuvim/test/__pycache__ 
+RUN rm -rf src/emuvim/test/unittests/__pycache__ 
+RUN rm -rf src/emuvim/test/integrationtests/__pycache__ 
+RUN python setup.py install 
+RUN echo 'Done'
+
 
 ENTRYPOINT ["/son-emu/utils/docker/entrypoint.sh"]
 
-# dummy GK, zerorpc, DCNetwork zerorpc, cAdvisor
-EXPOSE 5000 4242 5151 8090
+# dummy GK, zerorpc, DCNetwork zerorpc, cAdvisor, restAPI
+EXPOSE 5000 4242 5151 8090 5001
diff --git a/utils/docker/docker-compose_son-emu.yml b/utils/docker/docker-compose_son-emu.yml
new file mode 100755 (executable)
index 0000000..4e39e32
--- /dev/null
@@ -0,0 +1,19 @@
+version: '2'
+services:
+
+  son-emu:
+    image: registry.sonata-nfv.eu:5000/son-emu
+    container_name: son-emu
+    volumes:
+     - "/var/run/docker.sock:/var/run/docker.sock"
+    network_mode: "host"
+    pid: "host"
+    privileged: true
+    ports:
+     #REST api
+     - "5001:5001"
+    entrypoint: 
+     - "/son-emu/utils/ci/entrypoint_docker-compose.sh"
+    command: [python, src/emuvim/examples/son-monitor_default_topo.py]
+
+
index 1820515..4349efd 100755 (executable)
@@ -6,7 +6,7 @@ set -x
 
 # this cannot be done from the Dockerfile since we have the socket not mounted during build
 # this image is needed for the monitoring in son-emu
-echo 'Pulling the "google/cadvisor" image ... please wait'
-docker pull 'google/cadvisor'
+#echo 'Pulling the "google/cadvisor" image ... please wait'
+#docker pull 'google/cadvisor'
 
 exec /containernet/util/docker/entrypoint.sh $*
diff --git a/utils/docker/entrypoint_docker-compose.sh b/utils/docker/entrypoint_docker-compose.sh
new file mode 100755 (executable)
index 0000000..421444e
--- /dev/null
@@ -0,0 +1,68 @@
+#! /bin/bash -e
+
+# docker stop trap signals
+# https://medium.com/@gchudnov/trapping-signals-in-docker-containers-7a57fdda7d86#.5d6q01x7q
+
+pid=0
+command=""
+term_recvd=0
+
+# send SIGTERM also to the executed command in the docker container (the containernet topology)
+# SIGTERM-handler
+function term_handler() { 
+  echo $command        
+  pid=$(pgrep -f "$command" | sed -n 1p)
+
+  pid="$!"
+  # avoid that the process triggers its own handler by sending sigterm
+  if [ $pid -ne 0 ] && [ $term_recvd -eq 0 ]; then
+    echo "sigterm received"
+    echo $pid  
+    term_recvd=1
+    kill -SIGTERM "$pid"  
+  fi
+
+  wait "$pid"
+
+  # do some manual cleanup
+  # remove all containers started by son-emu
+  docker ps -a -q --filter="name=mn.*" | xargs -r docker rm -f
+  # cleanup remaining mininet
+  mn -c
+
+  sleep 5
+  exit 143; # 128 + 15 -- SIGTERM
+}
+
+# setup handlers
+# on callback, kill the last background process, which is `tail -f /dev/null` and execute the specified handler
+trap 'term_handler' SIGTERM
+
+
+service openvswitch-switch start
+
+if [ ! -S /var/run/docker.sock ]; then
+    echo 'Error: the Docker socket file "/var/run/docker.sock" was not found. It should be mounted as a volume.'
+    exit 1
+fi
+
+# this cannot be done from the Dockerfile since we have the socket not mounted during build
+echo 'Pulling the "ubuntu:trusty" image ... please wait'
+docker pull 'ubuntu:trusty'
+
+echo "Welcome to Containernet running within a Docker container ..."
+
+if [[ $# -eq 0 ]]; then
+    exec /bin/bash
+else
+    #remember command to send it also the SIGTERM via the handler
+    command=$*
+    echo $command      
+    exec $* &
+    # wait indefinetely
+    while true
+    do
+      sleep 1
+    done
+    echo "done"
+fi