From 68d042113554a310c357ef1b614c7a9e84cd7a72 Mon Sep 17 00:00:00 2001 From: schillinge Date: Mon, 11 Mar 2019 17:57:41 +0100 Subject: [PATCH] Ensure timely termination of all flask servers 1. The chain_api was never terminated. This was fixed by turning the server into an pyWSGI instance. 2. no monkey patching was applied. The monkey patching of gevent is required in order to be able to handle other events during a time.sleep() call. Since multiple patching is detected, it was added to all files which create WSGI servers. All in all this change fixes a large leak of threads, open files and performance. This change updates Containernet in order to fix race conditions which otherwise happen due to gevent's monkey patching. Change-Id: Ia45ad07db1f85046bfcac85eaca20c930b931141 Signed-off-by: schillinge --- Dockerfile | 2 +- src/emuvim/api/openstack/chain_api.py | 30 +++++++++++-------- src/emuvim/api/openstack/manage.py | 5 +++- .../api/openstack/openstack_api_endpoint.py | 14 ++++----- .../openstack_dummies/base_openstack_dummy.py | 18 +++++++++-- src/emuvim/api/rest/rest_api_endpoint.py | 3 ++ src/emuvim/test/base.py | 4 +++ 7 files changed, 49 insertions(+), 27 deletions(-) diff --git a/Dockerfile b/Dockerfile index b77c268..a15ddc9 100755 --- a/Dockerfile +++ b/Dockerfile @@ -49,7 +49,7 @@ RUN apt-get update \ # install containernet (using its Ansible playbook) # Attention: Containernet installation fixed to specific commit. Change to update to latest Containernet version. RUN git clone https://github.com/containernet/containernet.git && \ - (cd containernet && git checkout bc269d6f1cf9f50f71fda65c25fe1f2f4c1573b7) + (cd containernet && git checkout 6fcee82e192c8c0e6447650d6f512842185529ee) WORKDIR /containernet/ansible RUN ansible-playbook -i "localhost," -c local --skip-tags "notindocker" install.yml diff --git a/src/emuvim/api/openstack/chain_api.py b/src/emuvim/api/openstack/chain_api.py index 47af63c..fde3a42 100755 --- a/src/emuvim/api/openstack/chain_api.py +++ b/src/emuvim/api/openstack/chain_api.py @@ -27,12 +27,17 @@ import json import logging import copy +from gevent import monkey +from gevent.pywsgi import WSGIServer + from mininet.node import OVSSwitch from flask import Flask from flask import Response, request from flask_restful import Api, Resource +monkey.patch_all() + class ChainApi(Resource): """ @@ -65,7 +70,6 @@ class ChainApi(Resource): resource_class_kwargs={'api': self}) self.api.add_resource(QueryTopology, "/v1/topo", resource_class_kwargs={'api': self}) - self.api.add_resource(Shutdown, "/shutdown") @self.app.after_request def add_access_control_header(response): @@ -75,9 +79,18 @@ class ChainApi(Resource): def _start_flask(self): logging.info("Starting %s endpoint @ http://%s:%d" % ("ChainDummyApi", self.ip, self.port)) - if self.app is not None: - self.app.before_request(self.dump_playbook) - self.app.run(self.ip, self.port, debug=True, use_reloader=False) + self.http_server = WSGIServer( + (self.ip, self.port), + self.app, + log=open("/dev/null", "w") # don't show http logs + ) + self.http_server.serve_forever(stop_timeout=1) + logging.info('Stopped %s' % self.__class__.__name__) + + def stop(self): + if self.http_server: + logging.info('Stopping %s' % self.__class__.__name__) + self.http_server.stop(timeout=1) def dump_playbook(self): with self.manage.lock: @@ -90,15 +103,6 @@ class ChainApi(Resource): logfile.write(data + "\n") -class Shutdown(Resource): - def get(self): - logging.debug(("%s is beeing shut down") % (__name__)) - func = request.environ.get('werkzeug.server.shutdown') - if func is None: - raise RuntimeError('Not running with the Werkzeug Server') - func() - - class ChainVersionsList(Resource): ''' Entrypoint to find versions of the chain api. diff --git a/src/emuvim/api/openstack/manage.py b/src/emuvim/api/openstack/manage.py index 083550e..a78cb30 100755 --- a/src/emuvim/api/openstack/manage.py +++ b/src/emuvim/api/openstack/manage.py @@ -79,7 +79,6 @@ class OpenstackManage(object): # dependent! self.chain = chain_api.ChainApi(ip, port, self) self.thread = threading.Thread(target=self.chain._start_flask, args=()) - self.thread.daemon = True self.thread.name = self.chain.__class__ self.thread.start() @@ -92,6 +91,10 @@ class OpenstackManage(object): self.floating_intf = None self.floating_links = dict() + def stop(self): + self.chain.stop() + self.thread.join() + @property def net(self): return self._net diff --git a/src/emuvim/api/openstack/openstack_api_endpoint.py b/src/emuvim/api/openstack/openstack_api_endpoint.py index b6347eb..fdfa5e4 100755 --- a/src/emuvim/api/openstack/openstack_api_endpoint.py +++ b/src/emuvim/api/openstack/openstack_api_endpoint.py @@ -32,7 +32,6 @@ from openstack_dummies.neutron_dummy_api import NeutronDummyApi from openstack_dummies.nova_dummy_api import NovaDummyApi import logging -import threading import compute import socket import time @@ -99,10 +98,7 @@ class OpenstackApiEndpoint(): for c in self.openstack_endpoints.values(): c.compute = self.compute c.manage = self.manage - c.server_thread = threading.Thread(target=c._start_flask, args=()) - c.server_thread.daemon = True - c.server_thread.name = c.__class__.__name__ - c.server_thread.start() + c.start() if wait_for_port: self._wait_for_port(c.ip, c.port) @@ -112,10 +108,10 @@ class OpenstackApiEndpoint(): """ for c in self.openstack_endpoints.values(): c.stop() - # for c in self.openstack_endpoints.values(): - # if c.server_thread: - # print("Waiting for WSGIServers to be stopped ...") - # c.server_thread.join() + for c in self.openstack_endpoints.values(): + if c.server_thread: + c.server_thread.join() + self.manage.stop() def _wait_for_port(self, ip, port): for i in range(0, 10): diff --git a/src/emuvim/api/openstack/openstack_dummies/base_openstack_dummy.py b/src/emuvim/api/openstack/openstack_dummies/base_openstack_dummy.py index d8eeb79..57097d9 100755 --- a/src/emuvim/api/openstack/openstack_dummies/base_openstack_dummy.py +++ b/src/emuvim/api/openstack/openstack_dummies/base_openstack_dummy.py @@ -23,10 +23,15 @@ # the Horizon 2020 and 5G-PPP programmes. The authors would like to # acknowledge the contributions of their colleagues of the SONATA # partner consortium (www.sonata-nfv.eu). +import logging + +import threading from flask import Flask, request from flask_restful import Api, Resource +from gevent import monkey from gevent.pywsgi import WSGIServer -import logging + +monkey.patch_all() LOG = logging.getLogger("api.openstack.base") @@ -51,9 +56,15 @@ class BaseOpenstackDummy(Resource): self.app = Flask(__name__) self.api = Api(self.app) + def start(self): + self.server_thread = threading.Thread(target=self._start_flask, args=()) + self.server_thread.name = self.__class__.__name__ + self.server_thread.start() + def stop(self): if self.http_server: - self.http_server.stop(timeout=1.0) + LOG.info('Stopping %s' % self.__class__.__name__) + self.http_server.stop(timeout=1) def _start_flask(self): LOG.info("Starting %s endpoint @ http://%s:%d" % ( @@ -63,7 +74,8 @@ class BaseOpenstackDummy(Resource): self.app, log=open("/dev/null", "w") # don't show http logs ) - self.http_server.serve_forever(stop_timeout=1.0) + self.http_server.serve_forever(stop_timeout=1) + LOG.info('Stopped %s' % self.__class__.__name__) def dump_playbook(self): with self.manage.lock: diff --git a/src/emuvim/api/rest/rest_api_endpoint.py b/src/emuvim/api/rest/rest_api_endpoint.py index 4f9d6d8..a9a863d 100755 --- a/src/emuvim/api/rest/rest_api_endpoint.py +++ b/src/emuvim/api/rest/rest_api_endpoint.py @@ -28,6 +28,7 @@ import logging import threading from flask import Flask from flask_restful import Api +from gevent import monkey from gevent.pywsgi import WSGIServer # need to import total module to set its global variable dcs @@ -44,6 +45,8 @@ from monitor import MonitorInterfaceAction, MonitorFlowAction, MonitorLinkAction import pkg_resources from os import path +monkey.patch_all() + logging.basicConfig() diff --git a/src/emuvim/test/base.py b/src/emuvim/test/base.py index 6221765..9271f56 100755 --- a/src/emuvim/test/base.py +++ b/src/emuvim/test/base.py @@ -23,6 +23,10 @@ # the Horizon 2020 and 5G-PPP programmes. The authors would like to # acknowledge the contributions of their colleagues of the SONATA # partner consortium (www.sonata-nfv.eu). + +from gevent import monkey +monkey.patch_all() # noqa: because otherwise pep complains about code before imports + import unittest import os import subprocess -- 2.17.1