From: peusterm Date: Wed, 15 Nov 2017 07:23:50 +0000 (+0100) Subject: OpenStack API: Replaced Flask with WSGI X-Git-Tag: v4.0.0~10 X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2Fvim-emu.git;a=commitdiff_plain;h=4e5c81ecd37e7383c52dad55079f91b90dec22fe;hp=396e91e0dc05304d524a41d5a6f48d75c2867974 OpenStack API: Replaced Flask with WSGI Flask server introduced a couple of problems, e.g., pending sockets when an experiment was shut down. This patch solves this and improves stability and performance of the emulated OpenStack APIs. Increased PIP timeout in Dockerfile to make build more robust in CI environment. Change-Id: I54e040c56a10075555d13fe6fb144fc117340026 Signed-off-by: peusterm --- diff --git a/Dockerfile b/Dockerfile index 856d796..4b8627a 100755 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,7 @@ FROM ubuntu:xenial ENV SON_EMU_IN_DOCKER 1 +ENV PIP_DEFAULT_TIMEOUT=100 # install required packages RUN apt-get clean diff --git a/src/emuvim/api/openstack/openstack_api_endpoint.py b/src/emuvim/api/openstack/openstack_api_endpoint.py index fcdf9f0..e340a3a 100755 --- a/src/emuvim/api/openstack/openstack_api_endpoint.py +++ b/src/emuvim/api/openstack/openstack_api_endpoint.py @@ -88,28 +88,26 @@ class OpenstackApiEndpoint(): """ Start all connected OpenStack endpoints that are connected to this API endpoint. """ - for component in self.openstack_endpoints.values(): - component.compute = self.compute - component.manage = self.manage - thread = threading.Thread(target=component._start_flask, args=()) - thread.daemon = True - thread.name = component.__class__ - thread.start() + for c in self.openstack_endpoints.values(): + c.compute = self.compute + c.manage = self.manage + c.server_thread = threading.Thread(target=c._start_flask, args=()) + c.server_thread.daemon = True + c.server_thread.name = c.__class__.__name__ + c.server_thread.start() if wait_for_port: - self._wait_for_port(component.ip, component.port) - - + self._wait_for_port(c.ip, c.port) + def stop(self): """ Stop all connected OpenStack endpoints that are connected to this API endpoint. """ - for component in self.openstack_endpoints.values(): - url = "http://" + component.ip + ":" + str(component.port) + "/shutdown" - try: - requests.get(url) - except: - # seems to be stopped - pass + for c in self.openstack_endpoints.values(): + c.stop() + #for c in self.openstack_endpoints.values(): + # if c.server_thread: + # print("Waiting for WSGIServers to be stopped ...") + # c.server_thread.join() def _wait_for_port(self, ip, port): for i in range(0, 10): diff --git a/src/emuvim/api/openstack/openstack_dummies/base_openstack_dummy.py b/src/emuvim/api/openstack/openstack_dummies/base_openstack_dummy.py index d3c2483..93ad2bd 100755 --- a/src/emuvim/api/openstack/openstack_dummies/base_openstack_dummy.py +++ b/src/emuvim/api/openstack/openstack_dummies/base_openstack_dummy.py @@ -27,6 +27,7 @@ partner consortium (www.sonata-nfv.eu). """ from flask import Flask, request from flask_restful import Api, Resource +from gevent.pywsgi import WSGIServer import logging LOG = logging.getLogger("api.openstack.base") @@ -42,6 +43,8 @@ class BaseOpenstackDummy(Resource): self.port = port self.compute = None self.manage = None + self.http_server = None + self.server_thread = None self.playbook_file = '/tmp/son-emu-requests.log' with open(self.playbook_file, 'w'): pass @@ -50,11 +53,19 @@ class BaseOpenstackDummy(Resource): self.app = Flask(__name__) self.api = Api(self.app) + def stop(self): + if self.http_server: + self.http_server.stop(timeout=1.0) + def _start_flask(self): - LOG.info("Starting %s endpoint @ http://%s:%d" % (__name__, self.ip, self.port)) - if self.app is not None: - self.app.before_request(self.dump_playbook) - self.app.run(self.ip, self.port, debug=True, use_reloader=False) + LOG.info("Starting %s endpoint @ http://%s:%d" % ( + self.__class__.__name__, self.ip, self.port)) + self.http_server = WSGIServer( + (self.ip, self.port), + self.app, + log=open("/dev/null", "w") # don't show http logs + ) + self.http_server.serve_forever(stop_timeout=1.0) def dump_playbook(self): with self.manage.lock: diff --git a/src/emuvim/api/openstack/openstack_dummies/glance_dummy_api.py b/src/emuvim/api/openstack/openstack_dummies/glance_dummy_api.py index 5eb1100..4ce9e77 100755 --- a/src/emuvim/api/openstack/openstack_dummies/glance_dummy_api.py +++ b/src/emuvim/api/openstack/openstack_dummies/glance_dummy_api.py @@ -40,8 +40,6 @@ class GlanceDummyApi(BaseOpenstackDummy): def __init__(self, in_ip, in_port, compute): super(GlanceDummyApi, self).__init__(in_ip, in_port) self.compute = compute - self.api.add_resource(Shutdown, - "/shutdown") self.api.add_resource(GlanceListApiVersions, "/versions") self.api.add_resource(GlanceSchema, @@ -63,21 +61,6 @@ class GlanceDummyApi(BaseOpenstackDummy): "/v2/images//", resource_class_kwargs={'api': self}) - def _start_flask(self): - LOG.info("Starting %s endpoint @ http://%s:%d" % ("GlanceDummyApi", self.ip, self.port)) - if self.app is not None: - self.app.before_request(self.dump_playbook) - self.app.run(self.ip, self.port, debug=True, use_reloader=False) - - -class Shutdown(Resource): - def get(self): - LOG.debug(("%s is beeing shut down") % (__name__)) - func = request.environ.get('werkzeug.server.shutdown') - if func is None: - raise RuntimeError('Not running with the Werkzeug Server') - func() - class GlanceListApiVersions(Resource): def get(self): diff --git a/src/emuvim/api/openstack/openstack_dummies/heat_dummy_api.py b/src/emuvim/api/openstack/openstack_dummies/heat_dummy_api.py index e95d9d6..96af19a 100755 --- a/src/emuvim/api/openstack/openstack_dummies/heat_dummy_api.py +++ b/src/emuvim/api/openstack/openstack_dummies/heat_dummy_api.py @@ -44,7 +44,6 @@ class HeatDummyApi(BaseOpenstackDummy): super(HeatDummyApi, self).__init__(in_ip, in_port) self.compute = compute - self.api.add_resource(Shutdown, "/shutdown") self.api.add_resource(HeatListAPIVersions, "/", resource_class_kwargs={'api': self}) self.api.add_resource(HeatCreateStack, "/v1//stacks", @@ -69,26 +68,6 @@ class HeatDummyApi(BaseOpenstackDummy): return response - def _start_flask(self): - LOG.info("Starting %s endpoint @ http://%s:%d" % (__name__, self.ip, self.port)) - if self.app is not None: - self.app.before_request(self.dump_playbook) - self.app.run(self.ip, self.port, debug=True, use_reloader=False) - - -class Shutdown(Resource): - """ - A get request to /shutdown will shut down this endpoint. - """ - - def get(self): - LOG.debug(("%s is beeing shut down") % (__name__)) - func = request.environ.get('werkzeug.server.shutdown') - if func is None: - raise RuntimeError('Not running with the Werkzeug Server') - func() - - class HeatListAPIVersions(Resource): def __init__(self, api): self.api = api diff --git a/src/emuvim/api/openstack/openstack_dummies/keystone_dummy_api.py b/src/emuvim/api/openstack/openstack_dummies/keystone_dummy_api.py index ece2211..1a258df 100755 --- a/src/emuvim/api/openstack/openstack_dummies/keystone_dummy_api.py +++ b/src/emuvim/api/openstack/openstack_dummies/keystone_dummy_api.py @@ -40,31 +40,11 @@ class KeystoneDummyApi(BaseOpenstackDummy): super(KeystoneDummyApi, self).__init__(in_ip, in_port) self.api.add_resource(KeystoneListVersions, "/", resource_class_kwargs={'api': self}) - self.api.add_resource(Shutdown, "/shutdown") self.api.add_resource(KeystoneShowAPIv2, "/v2.0", resource_class_kwargs={'api': self}) self.api.add_resource(KeystoneGetToken, "/v2.0/tokens", resource_class_kwargs={'api': self}) self.api.add_resource(KeystoneShowAPIv3, "/v3.0", resource_class_kwargs={'api': self}) self.api.add_resource(KeystoneGetTokenv3, "/v3.0/auth/tokens", resource_class_kwargs={'api': self}) - def _start_flask(self): - LOG.info("Starting %s endpoint @ http://%s:%d" % (__name__, self.ip, self.port)) - if self.app is not None: - self.app.before_request(self.dump_playbook) - self.app.run(self.ip, self.port, debug=True, use_reloader=False) - - -class Shutdown(Resource): - """ - A get request to /shutdown will shut down this endpoint. - """ - - def get(self): - LOG.debug(("%s is beeing shut down") % (__name__)) - func = request.environ.get('werkzeug.server.shutdown') - if func is None: - raise RuntimeError('Not running with the Werkzeug Server') - func() - class KeystoneListVersions(Resource): """ diff --git a/src/emuvim/api/openstack/openstack_dummies/neutron_dummy_api.py b/src/emuvim/api/openstack/openstack_dummies/neutron_dummy_api.py index e0181d9..385163c 100755 --- a/src/emuvim/api/openstack/openstack_dummies/neutron_dummy_api.py +++ b/src/emuvim/api/openstack/openstack_dummies/neutron_dummy_api.py @@ -45,7 +45,6 @@ class NeutronDummyApi(BaseOpenstackDummy): self.compute = compute self.api.add_resource(NeutronListAPIVersions, "/") - self.api.add_resource(Shutdown, "/shutdown") self.api.add_resource(NeutronShowAPIv2Details, "/v2.0") self.api.add_resource(NeutronListNetworks, "/v2.0/networks.json", "/v2.0/networks", resource_class_kwargs={'api': self}) @@ -137,21 +136,6 @@ class NeutronDummyApi(BaseOpenstackDummy): "/v2.0/sfc/port_chains/", resource_class_kwargs={'api': self}) - def _start_flask(self): - LOG.info("Starting %s endpoint @ http://%s:%d" % (__name__, self.ip, self.port)) - if self.app is not None: - self.app.before_request(self.dump_playbook) - self.app.run(self.ip, self.port, debug=True, use_reloader=False) - - -class Shutdown(Resource): - def get(self): - LOG.debug(("%s is beeing shut down") % (__name__)) - func = request.environ.get('werkzeug.server.shutdown') - if func is None: - raise RuntimeError('Not running with the Werkzeug Server') - func() - class NeutronListAPIVersions(Resource): def get(self): diff --git a/src/emuvim/api/openstack/openstack_dummies/nova_dummy_api.py b/src/emuvim/api/openstack/openstack_dummies/nova_dummy_api.py index 7495076..5029417 100755 --- a/src/emuvim/api/openstack/openstack_dummies/nova_dummy_api.py +++ b/src/emuvim/api/openstack/openstack_dummies/nova_dummy_api.py @@ -42,10 +42,13 @@ class NovaDummyApi(BaseOpenstackDummy): def __init__(self, in_ip, in_port, compute): super(NovaDummyApi, self).__init__(in_ip, in_port) self.compute = compute + self.compute.add_flavor('m1.tiny', 1, 512, "MB", 1, "GB") + self.compute.add_flavor('m1.nano', 1, 64, "MB", 0, "GB") + self.compute.add_flavor('m1.micro', 1, 128, "MB", 0, "GB") + self.compute.add_flavor('m1.small', 1, 1024, "MB", 2, "GB") self.api.add_resource(NovaVersionsList, "/", resource_class_kwargs={'api': self}) - self.api.add_resource(Shutdown, "/shutdown") self.api.add_resource(NovaVersionShow, "/v2.1/", resource_class_kwargs={'api': self}) self.api.add_resource(NovaListServersApi, "/v2.1//servers", @@ -75,30 +78,6 @@ class NovaDummyApi(BaseOpenstackDummy): self.api.add_resource(NovaLimits, "/v2.1//limits", resource_class_kwargs={'api': self}) - def _start_flask(self): - LOG.info("Starting %s endpoint @ http://%s:%d" % ("NovaDummyApi", self.ip, self.port)) - # add some flavors for good measure - self.compute.add_flavor('m1.tiny', 1, 512, "MB", 1, "GB") - self.compute.add_flavor('m1.nano', 1, 64, "MB", 0, "GB") - self.compute.add_flavor('m1.micro', 1, 128, "MB", 0, "GB") - self.compute.add_flavor('m1.small', 1, 1024, "MB", 2, "GB") - if self.app is not None: - self.app.before_request(self.dump_playbook) - self.app.run(self.ip, self.port, debug=True, use_reloader=False) - - -class Shutdown(Resource): - """ - A get request to /shutdown will shut down this endpoint. - """ - - def get(self): - LOG.debug(("%s is beeing shut doen") % (__name__)) - func = request.environ.get('werkzeug.server.shutdown') - if func is None: - raise RuntimeError('Not running with the Werkzeug Server') - func() - class NovaVersionsList(Resource): def __init__(self, api):