OpenStack API: Replaced Flask with WSGI

Flask server introduced a couple of problems, e.g., pending
sockets when an experiment was shut down.
This patch solves this and improves stability and performance
of the emulated OpenStack APIs.

Increased PIP timeout in Dockerfile to make build more
robust in CI environment.

Change-Id: I54e040c56a10075555d13fe6fb144fc117340026
Signed-off-by: peusterm <manuel.peuster@uni-paderborn.de>

diff --git a/Dockerfile b/Dockerfile
index 856d796..4b8627a 100755 (executable)
--- a/Dockerfile
+++ b/Dockerfile
@@ -27,6 +27,7 @@
 FROM ubuntu:xenial
 
 ENV SON_EMU_IN_DOCKER 1
+ENV PIP_DEFAULT_TIMEOUT=100
 
 # install required packages
 RUN apt-get clean

diff --git a/src/emuvim/api/openstack/openstack_api_endpoint.py b/src/emuvim/api/openstack/openstack_api_endpoint.py
index fcdf9f0..e340a3a 100755 (executable)
--- a/src/emuvim/api/openstack/openstack_api_endpoint.py
+++ b/src/emuvim/api/openstack/openstack_api_endpoint.py
@@ -88,28 +88,26 @@ class OpenstackApiEndpoint():
         """
         Start all connected OpenStack endpoints that are connected to this API endpoint.
         """
-        for component in self.openstack_endpoints.values():
-            component.compute = self.compute
-            component.manage = self.manage
-            thread = threading.Thread(target=component._start_flask, args=())
-            thread.daemon = True
-            thread.name = component.__class__
-            thread.start()
+        for c in self.openstack_endpoints.values():
+            c.compute = self.compute
+            c.manage = self.manage
+            c.server_thread = threading.Thread(target=c._start_flask, args=())
+            c.server_thread.daemon = True
+            c.server_thread.name = c.__class__.__name__
+            c.server_thread.start()
             if wait_for_port:
-                self._wait_for_port(component.ip, component.port)
-                
-
+                self._wait_for_port(c.ip, c.port)
+       
     def stop(self):
         """
         Stop all connected OpenStack endpoints that are connected to this API endpoint.
         """
-        for component in self.openstack_endpoints.values():
-            url = "http://" + component.ip + ":" + str(component.port) + "/shutdown"
-            try:
-                requests.get(url)
-            except:
-                # seems to be stopped
-                pass
+        for c in self.openstack_endpoints.values():
+            c.stop()
+        #for c in self.openstack_endpoints.values():
+        #    if c.server_thread:
+        #        print("Waiting for WSGIServers to be stopped ...")
+        #        c.server_thread.join()
 
     def _wait_for_port(self, ip, port):
         for i in range(0, 10):

diff --git a/src/emuvim/api/openstack/openstack_dummies/base_openstack_dummy.py b/src/emuvim/api/openstack/openstack_dummies/base_openstack_dummy.py
index d3c2483..93ad2bd 100755 (executable)
--- a/src/emuvim/api/openstack/openstack_dummies/base_openstack_dummy.py
+++ b/src/emuvim/api/openstack/openstack_dummies/base_openstack_dummy.py
@@ -27,6 +27,7 @@ partner consortium (www.sonata-nfv.eu).
 """
 from flask import Flask, request
 from flask_restful import Api, Resource
+from gevent.pywsgi import WSGIServer
 import logging
 
 LOG = logging.getLogger("api.openstack.base")
@@ -42,6 +43,8 @@ class BaseOpenstackDummy(Resource):
         self.port = port
         self.compute = None
         self.manage = None
+        self.http_server = None
+        self.server_thread = None
         self.playbook_file = '/tmp/son-emu-requests.log'
         with open(self.playbook_file, 'w'):
             pass
@@ -50,11 +53,19 @@ class BaseOpenstackDummy(Resource):
         self.app = Flask(__name__)
         self.api = Api(self.app)
 
+    def stop(self):
+        if self.http_server:
+            self.http_server.stop(timeout=1.0)
+
     def _start_flask(self):
-        LOG.info("Starting %s endpoint @ http://%s:%d" % (__name__, self.ip, self.port))
-        if self.app is not None:
-            self.app.before_request(self.dump_playbook)
-            self.app.run(self.ip, self.port, debug=True, use_reloader=False)
+        LOG.info("Starting %s endpoint @ http://%s:%d" % (
+            self.__class__.__name__, self.ip, self.port))
+        self.http_server = WSGIServer(
+            (self.ip, self.port),
+            self.app,
+            log=open("/dev/null", "w")  # don't show http logs
+        )
+        self.http_server.serve_forever(stop_timeout=1.0)
 
     def dump_playbook(self):
         with self.manage.lock:

diff --git a/src/emuvim/api/openstack/openstack_dummies/glance_dummy_api.py b/src/emuvim/api/openstack/openstack_dummies/glance_dummy_api.py
index 5eb1100..4ce9e77 100755 (executable)
--- a/src/emuvim/api/openstack/openstack_dummies/glance_dummy_api.py
+++ b/src/emuvim/api/openstack/openstack_dummies/glance_dummy_api.py
@@ -40,8 +40,6 @@ class GlanceDummyApi(BaseOpenstackDummy):
     def __init__(self, in_ip, in_port, compute):
         super(GlanceDummyApi, self).__init__(in_ip, in_port)
         self.compute = compute
-        self.api.add_resource(Shutdown,
-                              "/shutdown")
         self.api.add_resource(GlanceListApiVersions,
                               "/versions")
         self.api.add_resource(GlanceSchema,
@@ -63,21 +61,6 @@ class GlanceDummyApi(BaseOpenstackDummy):
                               "/v2/images/<owner>/<container>",
                               resource_class_kwargs={'api': self})
 
-    def _start_flask(self):
-        LOG.info("Starting %s endpoint @ http://%s:%d" % ("GlanceDummyApi", self.ip, self.port))
-        if self.app is not None:
-            self.app.before_request(self.dump_playbook)
-            self.app.run(self.ip, self.port, debug=True, use_reloader=False)
-
-
-class Shutdown(Resource):
-    def get(self):
-        LOG.debug(("%s is beeing shut down") % (__name__))
-        func = request.environ.get('werkzeug.server.shutdown')
-        if func is None:
-            raise RuntimeError('Not running with the Werkzeug Server')
-        func()
-
 
 class GlanceListApiVersions(Resource):
     def get(self):

diff --git a/src/emuvim/api/openstack/openstack_dummies/heat_dummy_api.py b/src/emuvim/api/openstack/openstack_dummies/heat_dummy_api.py
index e95d9d6..96af19a 100755 (executable)
--- a/src/emuvim/api/openstack/openstack_dummies/heat_dummy_api.py
+++ b/src/emuvim/api/openstack/openstack_dummies/heat_dummy_api.py
@@ -44,7 +44,6 @@ class HeatDummyApi(BaseOpenstackDummy):
         super(HeatDummyApi, self).__init__(in_ip, in_port)
         self.compute = compute
 
-        self.api.add_resource(Shutdown, "/shutdown")
         self.api.add_resource(HeatListAPIVersions, "/",
                               resource_class_kwargs={'api': self})
         self.api.add_resource(HeatCreateStack, "/v1/<tenant_id>/stacks",
@@ -69,26 +68,6 @@ class HeatDummyApi(BaseOpenstackDummy):
             return response
 
 
-    def _start_flask(self):
-        LOG.info("Starting %s endpoint @ http://%s:%d" % (__name__, self.ip, self.port))
-        if self.app is not None:
-            self.app.before_request(self.dump_playbook)
-            self.app.run(self.ip, self.port, debug=True, use_reloader=False)
-
-
-class Shutdown(Resource):
-    """
-    A get request to /shutdown will shut down this endpoint.
-    """
-
-    def get(self):
-        LOG.debug(("%s is beeing shut down") % (__name__))
-        func = request.environ.get('werkzeug.server.shutdown')
-        if func is None:
-            raise RuntimeError('Not running with the Werkzeug Server')
-        func()
-
-
 class HeatListAPIVersions(Resource):
     def __init__(self, api):
         self.api = api

diff --git a/src/emuvim/api/openstack/openstack_dummies/keystone_dummy_api.py b/src/emuvim/api/openstack/openstack_dummies/keystone_dummy_api.py
index ece2211..1a258df 100755 (executable)
--- a/src/emuvim/api/openstack/openstack_dummies/keystone_dummy_api.py
+++ b/src/emuvim/api/openstack/openstack_dummies/keystone_dummy_api.py
@@ -40,31 +40,11 @@ class KeystoneDummyApi(BaseOpenstackDummy):
         super(KeystoneDummyApi, self).__init__(in_ip, in_port)
 
         self.api.add_resource(KeystoneListVersions, "/", resource_class_kwargs={'api': self})
-        self.api.add_resource(Shutdown, "/shutdown")
         self.api.add_resource(KeystoneShowAPIv2, "/v2.0", resource_class_kwargs={'api': self})
         self.api.add_resource(KeystoneGetToken, "/v2.0/tokens", resource_class_kwargs={'api': self})
         self.api.add_resource(KeystoneShowAPIv3, "/v3.0", resource_class_kwargs={'api': self})
         self.api.add_resource(KeystoneGetTokenv3, "/v3.0/auth/tokens", resource_class_kwargs={'api': self})
 
-    def _start_flask(self):
-        LOG.info("Starting %s endpoint @ http://%s:%d" % (__name__, self.ip, self.port))
-        if self.app is not None:
-            self.app.before_request(self.dump_playbook)
-            self.app.run(self.ip, self.port, debug=True, use_reloader=False)
-
-
-class Shutdown(Resource):
-    """
-    A get request to /shutdown will shut down this endpoint.
-    """
-
-    def get(self):
-        LOG.debug(("%s is beeing shut down") % (__name__))
-        func = request.environ.get('werkzeug.server.shutdown')
-        if func is None:
-            raise RuntimeError('Not running with the Werkzeug Server')
-        func()
-
 
 class KeystoneListVersions(Resource):
     """

diff --git a/src/emuvim/api/openstack/openstack_dummies/neutron_dummy_api.py b/src/emuvim/api/openstack/openstack_dummies/neutron_dummy_api.py
index e0181d9..385163c 100755 (executable)
--- a/src/emuvim/api/openstack/openstack_dummies/neutron_dummy_api.py
+++ b/src/emuvim/api/openstack/openstack_dummies/neutron_dummy_api.py
@@ -45,7 +45,6 @@ class NeutronDummyApi(BaseOpenstackDummy):
         self.compute = compute
 
         self.api.add_resource(NeutronListAPIVersions, "/")
-        self.api.add_resource(Shutdown, "/shutdown")
         self.api.add_resource(NeutronShowAPIv2Details, "/v2.0")
         self.api.add_resource(NeutronListNetworks, "/v2.0/networks.json", "/v2.0/networks",
                               resource_class_kwargs={'api': self})
@@ -137,21 +136,6 @@ class NeutronDummyApi(BaseOpenstackDummy):
                               "/v2.0/sfc/port_chains/<chain_id>",
                               resource_class_kwargs={'api': self})
 
-    def _start_flask(self):
-        LOG.info("Starting %s endpoint @ http://%s:%d" % (__name__, self.ip, self.port))
-        if self.app is not None:
-            self.app.before_request(self.dump_playbook)
-            self.app.run(self.ip, self.port, debug=True, use_reloader=False)
-
-
-class Shutdown(Resource):
-    def get(self):
-        LOG.debug(("%s is beeing shut down") % (__name__))
-        func = request.environ.get('werkzeug.server.shutdown')
-        if func is None:
-            raise RuntimeError('Not running with the Werkzeug Server')
-        func()
-
 
 class NeutronListAPIVersions(Resource):
     def get(self):

diff --git a/src/emuvim/api/openstack/openstack_dummies/nova_dummy_api.py b/src/emuvim/api/openstack/openstack_dummies/nova_dummy_api.py
index 7495076..5029417 100755 (executable)
--- a/src/emuvim/api/openstack/openstack_dummies/nova_dummy_api.py
+++ b/src/emuvim/api/openstack/openstack_dummies/nova_dummy_api.py
@@ -42,10 +42,13 @@ class NovaDummyApi(BaseOpenstackDummy):
     def __init__(self, in_ip, in_port, compute):
         super(NovaDummyApi, self).__init__(in_ip, in_port)
         self.compute = compute
+        self.compute.add_flavor('m1.tiny', 1, 512, "MB", 1, "GB")
+        self.compute.add_flavor('m1.nano', 1, 64, "MB", 0, "GB")
+        self.compute.add_flavor('m1.micro', 1, 128, "MB", 0, "GB")
+        self.compute.add_flavor('m1.small', 1, 1024, "MB", 2, "GB")
 
         self.api.add_resource(NovaVersionsList, "/",
                               resource_class_kwargs={'api': self})
-        self.api.add_resource(Shutdown, "/shutdown")
         self.api.add_resource(NovaVersionShow, "/v2.1/<id>",
                               resource_class_kwargs={'api': self})
         self.api.add_resource(NovaListServersApi, "/v2.1/<id>/servers",
@@ -75,30 +78,6 @@ class NovaDummyApi(BaseOpenstackDummy):
         self.api.add_resource(NovaLimits, "/v2.1/<id>/limits",
                               resource_class_kwargs={'api': self})
 
-    def _start_flask(self):
-        LOG.info("Starting %s endpoint @ http://%s:%d" % ("NovaDummyApi", self.ip, self.port))
-        # add some flavors for good measure
-        self.compute.add_flavor('m1.tiny', 1, 512, "MB", 1, "GB")
-        self.compute.add_flavor('m1.nano', 1, 64, "MB", 0, "GB")
-        self.compute.add_flavor('m1.micro', 1, 128, "MB", 0, "GB")
-        self.compute.add_flavor('m1.small', 1, 1024, "MB", 2, "GB")
-        if self.app is not None:
-            self.app.before_request(self.dump_playbook)
-            self.app.run(self.ip, self.port, debug=True, use_reloader=False)
-
-
-class Shutdown(Resource):
-    """
-    A get request to /shutdown will shut down this endpoint.
-    """
-
-    def get(self):
-        LOG.debug(("%s is beeing shut doen") % (__name__))
-        func = request.environ.get('werkzeug.server.shutdown')
-        if func is None:
-            raise RuntimeError('Not running with the Werkzeug Server')
-        func()
-
 
 class NovaVersionsList(Resource):
     def __init__(self, api):