Ensure timely termination of all flask servers

1. The chain_api was never terminated. This was fixed by turning the
server into an pyWSGI instance.

2. no monkey patching was applied. The monkey patching of gevent is
required in order to be able to handle other events during a
time.sleep() call. Since multiple patching is detected, it was added to
all files which create WSGI servers.

All in all this change fixes a large leak of threads, open files and
performance.

This change updates Containernet in order to fix race conditions which
otherwise happen due to gevent's monkey patching.

Change-Id: Ia45ad07db1f85046bfcac85eaca20c930b931141
Signed-off-by: schillinge <ablu@mail.uni-paderborn.de>

diff --git a/Dockerfile b/Dockerfile
index b77c268..a15ddc9 100755 (executable)
--- a/Dockerfile
+++ b/Dockerfile
@@ -49,7 +49,7 @@ RUN apt-get update \
 # install containernet (using its Ansible playbook)
 # Attention: Containernet installation fixed to specific commit. Change to update to latest Containernet version.
 RUN git clone https://github.com/containernet/containernet.git && \
-    (cd containernet && git checkout bc269d6f1cf9f50f71fda65c25fe1f2f4c1573b7)
+    (cd containernet && git checkout 6fcee82e192c8c0e6447650d6f512842185529ee)
 WORKDIR /containernet/ansible
 RUN ansible-playbook -i "localhost," -c local --skip-tags "notindocker" install.yml
 

diff --git a/src/emuvim/api/openstack/chain_api.py b/src/emuvim/api/openstack/chain_api.py
index 47af63c..fde3a42 100755 (executable)
--- a/src/emuvim/api/openstack/chain_api.py
+++ b/src/emuvim/api/openstack/chain_api.py
@@ -27,12 +27,17 @@ import json
 import logging
 import copy
 
+from gevent import monkey
+from gevent.pywsgi import WSGIServer
+
 from mininet.node import OVSSwitch
 
 from flask import Flask
 from flask import Response, request
 from flask_restful import Api, Resource
 
+monkey.patch_all()
+
 
 class ChainApi(Resource):
     """
@@ -65,7 +70,6 @@ class ChainApi(Resource):
                               resource_class_kwargs={'api': self})
         self.api.add_resource(QueryTopology, "/v1/topo",
                               resource_class_kwargs={'api': self})
-        self.api.add_resource(Shutdown, "/shutdown")
 
         @self.app.after_request
         def add_access_control_header(response):
@@ -75,9 +79,18 @@ class ChainApi(Resource):
     def _start_flask(self):
         logging.info("Starting %s endpoint @ http://%s:%d" %
                      ("ChainDummyApi", self.ip, self.port))
-        if self.app is not None:
-            self.app.before_request(self.dump_playbook)
-            self.app.run(self.ip, self.port, debug=True, use_reloader=False)
+        self.http_server = WSGIServer(
+            (self.ip, self.port),
+            self.app,
+            log=open("/dev/null", "w")  # don't show http logs
+        )
+        self.http_server.serve_forever(stop_timeout=1)
+        logging.info('Stopped %s' % self.__class__.__name__)
+
+    def stop(self):
+        if self.http_server:
+            logging.info('Stopping %s' % self.__class__.__name__)
+            self.http_server.stop(timeout=1)
 
     def dump_playbook(self):
         with self.manage.lock:
@@ -90,15 +103,6 @@ class ChainApi(Resource):
                     logfile.write(data + "\n")
 
 
-class Shutdown(Resource):
-    def get(self):
-        logging.debug(("%s is beeing shut down") % (__name__))
-        func = request.environ.get('werkzeug.server.shutdown')
-        if func is None:
-            raise RuntimeError('Not running with the Werkzeug Server')
-        func()
-
-
 class ChainVersionsList(Resource):
     '''
     Entrypoint to find versions of the chain api.

diff --git a/src/emuvim/api/openstack/manage.py b/src/emuvim/api/openstack/manage.py
index 083550e..a78cb30 100755 (executable)
--- a/src/emuvim/api/openstack/manage.py
+++ b/src/emuvim/api/openstack/manage.py
@@ -79,7 +79,6 @@ class OpenstackManage(object):
         # dependent!
         self.chain = chain_api.ChainApi(ip, port, self)
         self.thread = threading.Thread(target=self.chain._start_flask, args=())
-        self.thread.daemon = True
         self.thread.name = self.chain.__class__
         self.thread.start()
 
@@ -92,6 +91,10 @@ class OpenstackManage(object):
         self.floating_intf = None
         self.floating_links = dict()
 
+    def stop(self):
+        self.chain.stop()
+        self.thread.join()
+
     @property
     def net(self):
         return self._net

diff --git a/src/emuvim/api/openstack/openstack_api_endpoint.py b/src/emuvim/api/openstack/openstack_api_endpoint.py
index b6347eb..fdfa5e4 100755 (executable)
--- a/src/emuvim/api/openstack/openstack_api_endpoint.py
+++ b/src/emuvim/api/openstack/openstack_api_endpoint.py
@@ -32,7 +32,6 @@ from openstack_dummies.neutron_dummy_api import NeutronDummyApi
 from openstack_dummies.nova_dummy_api import NovaDummyApi
 
 import logging
-import threading
 import compute
 import socket
 import time
@@ -99,10 +98,7 @@ class OpenstackApiEndpoint():
         for c in self.openstack_endpoints.values():
             c.compute = self.compute
             c.manage = self.manage
-            c.server_thread = threading.Thread(target=c._start_flask, args=())
-            c.server_thread.daemon = True
-            c.server_thread.name = c.__class__.__name__
-            c.server_thread.start()
+            c.start()
             if wait_for_port:
                 self._wait_for_port(c.ip, c.port)
 
@@ -112,10 +108,10 @@ class OpenstackApiEndpoint():
         """
         for c in self.openstack_endpoints.values():
             c.stop()
-        # for c in self.openstack_endpoints.values():
-        #    if c.server_thread:
-        #        print("Waiting for WSGIServers to be stopped ...")
-        #        c.server_thread.join()
+        for c in self.openstack_endpoints.values():
+            if c.server_thread:
+                c.server_thread.join()
+        self.manage.stop()
 
     def _wait_for_port(self, ip, port):
         for i in range(0, 10):

diff --git a/src/emuvim/api/openstack/openstack_dummies/base_openstack_dummy.py b/src/emuvim/api/openstack/openstack_dummies/base_openstack_dummy.py
index d8eeb79..57097d9 100755 (executable)
--- a/src/emuvim/api/openstack/openstack_dummies/base_openstack_dummy.py
+++ b/src/emuvim/api/openstack/openstack_dummies/base_openstack_dummy.py
@@ -23,10 +23,15 @@
 # the Horizon 2020 and 5G-PPP programmes. The authors would like to
 # acknowledge the contributions of their colleagues of the SONATA
 # partner consortium (www.sonata-nfv.eu).
+import logging
+
+import threading
 from flask import Flask, request
 from flask_restful import Api, Resource
+from gevent import monkey
 from gevent.pywsgi import WSGIServer
-import logging
+
+monkey.patch_all()
 
 LOG = logging.getLogger("api.openstack.base")
 
@@ -51,9 +56,15 @@ class BaseOpenstackDummy(Resource):
         self.app = Flask(__name__)
         self.api = Api(self.app)
 
+    def start(self):
+        self.server_thread = threading.Thread(target=self._start_flask, args=())
+        self.server_thread.name = self.__class__.__name__
+        self.server_thread.start()
+
     def stop(self):
         if self.http_server:
-            self.http_server.stop(timeout=1.0)
+            LOG.info('Stopping %s' % self.__class__.__name__)
+            self.http_server.stop(timeout=1)
 
     def _start_flask(self):
         LOG.info("Starting %s endpoint @ http://%s:%d" % (
@@ -63,7 +74,8 @@ class BaseOpenstackDummy(Resource):
             self.app,
             log=open("/dev/null", "w")  # don't show http logs
         )
-        self.http_server.serve_forever(stop_timeout=1.0)
+        self.http_server.serve_forever(stop_timeout=1)
+        LOG.info('Stopped %s' % self.__class__.__name__)
 
     def dump_playbook(self):
         with self.manage.lock:

diff --git a/src/emuvim/api/rest/rest_api_endpoint.py b/src/emuvim/api/rest/rest_api_endpoint.py
index 4f9d6d8..a9a863d 100755 (executable)
--- a/src/emuvim/api/rest/rest_api_endpoint.py
+++ b/src/emuvim/api/rest/rest_api_endpoint.py
@@ -28,6 +28,7 @@ import logging
 import threading
 from flask import Flask
 from flask_restful import Api
+from gevent import monkey
 from gevent.pywsgi import WSGIServer
 
 # need to import total module to set its global variable dcs
@@ -44,6 +45,8 @@ from monitor import MonitorInterfaceAction, MonitorFlowAction, MonitorLinkAction
 import pkg_resources
 from os import path
 
+monkey.patch_all()
+
 logging.basicConfig()
 
 

diff --git a/src/emuvim/test/base.py b/src/emuvim/test/base.py
index 6221765..9271f56 100755 (executable)
--- a/src/emuvim/test/base.py
+++ b/src/emuvim/test/base.py
@@ -23,6 +23,10 @@
 # the Horizon 2020 and 5G-PPP programmes. The authors would like to
 # acknowledge the contributions of their colleagues of the SONATA
 # partner consortium (www.sonata-nfv.eu).
+
+from gevent import monkey
+monkey.patch_all()  # noqa: because otherwise pep complains about code before imports
+
 import unittest
 import os
 import subprocess