Ensure timely termination of all flask servers
1. The chain_api was never terminated. This was fixed by turning the
server into an pyWSGI instance.
2. no monkey patching was applied. The monkey patching of gevent is
required in order to be able to handle other events during a
time.sleep() call. Since multiple patching is detected, it was added to
all files which create WSGI servers.
All in all this change fixes a large leak of threads, open files and
performance.
This change updates Containernet in order to fix race conditions which
otherwise happen due to gevent's monkey patching.
Change-Id: Ia45ad07db1f85046bfcac85eaca20c930b931141
Signed-off-by: schillinge <ablu@mail.uni-paderborn.de>
diff --git a/src/emuvim/api/openstack/chain_api.py b/src/emuvim/api/openstack/chain_api.py
index 47af63c..fde3a42 100755
--- a/src/emuvim/api/openstack/chain_api.py
+++ b/src/emuvim/api/openstack/chain_api.py
@@ -27,12 +27,17 @@
import logging
import copy
+from gevent import monkey
+from gevent.pywsgi import WSGIServer
+
from mininet.node import OVSSwitch
from flask import Flask
from flask import Response, request
from flask_restful import Api, Resource
+monkey.patch_all()
+
class ChainApi(Resource):
"""
@@ -65,7 +70,6 @@
resource_class_kwargs={'api': self})
self.api.add_resource(QueryTopology, "/v1/topo",
resource_class_kwargs={'api': self})
- self.api.add_resource(Shutdown, "/shutdown")
@self.app.after_request
def add_access_control_header(response):
@@ -75,9 +79,18 @@
def _start_flask(self):
logging.info("Starting %s endpoint @ http://%s:%d" %
("ChainDummyApi", self.ip, self.port))
- if self.app is not None:
- self.app.before_request(self.dump_playbook)
- self.app.run(self.ip, self.port, debug=True, use_reloader=False)
+ self.http_server = WSGIServer(
+ (self.ip, self.port),
+ self.app,
+ log=open("/dev/null", "w") # don't show http logs
+ )
+ self.http_server.serve_forever(stop_timeout=1)
+ logging.info('Stopped %s' % self.__class__.__name__)
+
+ def stop(self):
+ if self.http_server:
+ logging.info('Stopping %s' % self.__class__.__name__)
+ self.http_server.stop(timeout=1)
def dump_playbook(self):
with self.manage.lock:
@@ -90,15 +103,6 @@
logfile.write(data + "\n")
-class Shutdown(Resource):
- def get(self):
- logging.debug(("%s is beeing shut down") % (__name__))
- func = request.environ.get('werkzeug.server.shutdown')
- if func is None:
- raise RuntimeError('Not running with the Werkzeug Server')
- func()
-
-
class ChainVersionsList(Resource):
'''
Entrypoint to find versions of the chain api.
diff --git a/src/emuvim/api/openstack/manage.py b/src/emuvim/api/openstack/manage.py
index 083550e..a78cb30 100755
--- a/src/emuvim/api/openstack/manage.py
+++ b/src/emuvim/api/openstack/manage.py
@@ -79,7 +79,6 @@
# dependent!
self.chain = chain_api.ChainApi(ip, port, self)
self.thread = threading.Thread(target=self.chain._start_flask, args=())
- self.thread.daemon = True
self.thread.name = self.chain.__class__
self.thread.start()
@@ -92,6 +91,10 @@
self.floating_intf = None
self.floating_links = dict()
+ def stop(self):
+ self.chain.stop()
+ self.thread.join()
+
@property
def net(self):
return self._net
diff --git a/src/emuvim/api/openstack/openstack_api_endpoint.py b/src/emuvim/api/openstack/openstack_api_endpoint.py
index b6347eb..fdfa5e4 100755
--- a/src/emuvim/api/openstack/openstack_api_endpoint.py
+++ b/src/emuvim/api/openstack/openstack_api_endpoint.py
@@ -32,7 +32,6 @@
from openstack_dummies.nova_dummy_api import NovaDummyApi
import logging
-import threading
import compute
import socket
import time
@@ -99,10 +98,7 @@
for c in self.openstack_endpoints.values():
c.compute = self.compute
c.manage = self.manage
- c.server_thread = threading.Thread(target=c._start_flask, args=())
- c.server_thread.daemon = True
- c.server_thread.name = c.__class__.__name__
- c.server_thread.start()
+ c.start()
if wait_for_port:
self._wait_for_port(c.ip, c.port)
@@ -112,10 +108,10 @@
"""
for c in self.openstack_endpoints.values():
c.stop()
- # for c in self.openstack_endpoints.values():
- # if c.server_thread:
- # print("Waiting for WSGIServers to be stopped ...")
- # c.server_thread.join()
+ for c in self.openstack_endpoints.values():
+ if c.server_thread:
+ c.server_thread.join()
+ self.manage.stop()
def _wait_for_port(self, ip, port):
for i in range(0, 10):
diff --git a/src/emuvim/api/openstack/openstack_dummies/base_openstack_dummy.py b/src/emuvim/api/openstack/openstack_dummies/base_openstack_dummy.py
index d8eeb79..57097d9 100755
--- a/src/emuvim/api/openstack/openstack_dummies/base_openstack_dummy.py
+++ b/src/emuvim/api/openstack/openstack_dummies/base_openstack_dummy.py
@@ -23,10 +23,15 @@
# the Horizon 2020 and 5G-PPP programmes. The authors would like to
# acknowledge the contributions of their colleagues of the SONATA
# partner consortium (www.sonata-nfv.eu).
+import logging
+
+import threading
from flask import Flask, request
from flask_restful import Api, Resource
+from gevent import monkey
from gevent.pywsgi import WSGIServer
-import logging
+
+monkey.patch_all()
LOG = logging.getLogger("api.openstack.base")
@@ -51,9 +56,15 @@
self.app = Flask(__name__)
self.api = Api(self.app)
+ def start(self):
+ self.server_thread = threading.Thread(target=self._start_flask, args=())
+ self.server_thread.name = self.__class__.__name__
+ self.server_thread.start()
+
def stop(self):
if self.http_server:
- self.http_server.stop(timeout=1.0)
+ LOG.info('Stopping %s' % self.__class__.__name__)
+ self.http_server.stop(timeout=1)
def _start_flask(self):
LOG.info("Starting %s endpoint @ http://%s:%d" % (
@@ -63,7 +74,8 @@
self.app,
log=open("/dev/null", "w") # don't show http logs
)
- self.http_server.serve_forever(stop_timeout=1.0)
+ self.http_server.serve_forever(stop_timeout=1)
+ LOG.info('Stopped %s' % self.__class__.__name__)
def dump_playbook(self):
with self.manage.lock:
diff --git a/src/emuvim/api/rest/rest_api_endpoint.py b/src/emuvim/api/rest/rest_api_endpoint.py
index 4f9d6d8..a9a863d 100755
--- a/src/emuvim/api/rest/rest_api_endpoint.py
+++ b/src/emuvim/api/rest/rest_api_endpoint.py
@@ -28,6 +28,7 @@
import threading
from flask import Flask
from flask_restful import Api
+from gevent import monkey
from gevent.pywsgi import WSGIServer
# need to import total module to set its global variable dcs
@@ -44,6 +45,8 @@
import pkg_resources
from os import path
+monkey.patch_all()
+
logging.basicConfig()
diff --git a/src/emuvim/test/base.py b/src/emuvim/test/base.py
index 6221765..9271f56 100755
--- a/src/emuvim/test/base.py
+++ b/src/emuvim/test/base.py
@@ -23,6 +23,10 @@
# the Horizon 2020 and 5G-PPP programmes. The authors would like to
# acknowledge the contributions of their colleagues of the SONATA
# partner consortium (www.sonata-nfv.eu).
+
+from gevent import monkey
+monkey.patch_all() # noqa: because otherwise pep complains about code before imports
+
import unittest
import os
import subprocess