src/emuvim/api/sonata/dummygatekeeper.py

   1 """
   2 This module implements a simple REST API that behaves like SONATA's gatekeeper.
   3
   4 It is only used to support the development of SONATA's SDK tools and to demonstrate
   5 the year 1 version of the emulator until the integration with WP4's orchestrator is done.
   6 """
   7
   8 import logging
   9 import os
  10 import uuid
  11 import hashlib
  12 import zipfile
  13 import yaml
  14 from docker import Client as DockerClient
  15 from flask import Flask, request
  16 import flask_restful as fr
  17
  18 LOG = logging.getLogger("sonata-dummy-gatekeeper")
  19 LOG.setLevel(logging.DEBUG)
  20 logging.getLogger("werkzeug").setLevel(logging.WARNING)
  21
  22
  23 UPLOAD_FOLDER = "/tmp/son-dummy-gk/uploads/"
  24 CATALOG_FOLDER = "/tmp/son-dummy-gk/catalog/"
  25
  26
  27 class Gatekeeper(object):
  28
  29     def __init__(self):
  30         self.services = dict()
  31         LOG.info("Create SONATA dummy gatekeeper.")
  32
  33     def register_service_package(self, service_uuid, service):
  34         """
  35         register new service package
  36         :param service_uuid
  37         :param service object
  38         """
  39         self.services[service_uuid] = service
  40         # lets perform all steps needed to onboard the service
  41         service.onboard()
  42
  43
  44 class Service(object):
  45     """
  46     This class represents a NS uploaded as a *.son package to the
  47     dummy gatekeeper.
  48     Can have multiple running instances of this service.
  49     """
  50
  51     def __init__(self,
  52                  service_uuid,
  53                  package_file_hash,
  54                  package_file_path):
  55         self.uuid = service_uuid
  56         self.package_file_hash = package_file_hash
  57         self.package_file_path = package_file_path
  58         self.package_content_path = os.path.join(CATALOG_FOLDER, "services/%s" % self.uuid)
  59         self.manifest = None
  60         self.nsd = None
  61         self.vnfds = dict()
  62         self.local_docker_files = dict()
  63         self.instances = dict()
  64
  65     def start_service(self, service_uuid):
  66         # TODO implement method
  67         # 1. parse descriptors
  68         # 2. do the corresponding dc.startCompute(name="foobar") calls
  69         # 3. store references to the compute objects in self.instantiations
  70         pass
  71
  72     def onboard(self):
  73         """
  74         Do all steps to prepare this service to be instantiated
  75         :return:
  76         """
  77         # 1. extract the contents of the package and store them in our catalog
  78         self._unpack_service_package()
  79         # 2. read in all descriptor files
  80         self._load_package_descriptor()
  81         self._load_nsd()
  82         self._load_vnfd()
  83         self._load_docker_files()
  84         # 3. prepare container images (e.g. download or build Dockerfile)
  85         self._build_images_from_dockerfiles()
  86         self._download_predefined_dockerimages()
  87
  88         LOG.info("On-boarded service: %r" % self.manifest.get("package_name"))
  89
  90     def _unpack_service_package(self):
  91         """
  92         unzip *.son file and store contents in CATALOG_FOLDER/services/<service_uuid>/
  93         """
  94         with zipfile.ZipFile(self.package_file_path, "r") as z:
  95             z.extractall(self.package_content_path)
  96
  97     def _load_package_descriptor(self):
  98         """
  99         Load the main package descriptor YAML and keep it as dict.
 100         :return:
 101         """
 102         self.manifest = load_yaml(
 103             os.path.join(
 104                 self.package_content_path, "META-INF/MANIFEST.MF"))
 105
 106     def _load_nsd(self):
 107         """
 108         Load the entry NSD YAML and keep it as dict.
 109         :return:
 110         """
 111         if "entry_service_template" in self.manifest:
 112             nsd_path = os.path.join(
 113                 self.package_content_path,
 114                 make_relative_path(self.manifest.get("entry_service_template")))
 115             self.nsd = load_yaml(nsd_path)
 116             LOG.debug("Loaded NSD: %r" % self.nsd.get("ns_name"))
 117
 118     def _load_vnfd(self):
 119         """
 120         Load all VNFD YAML files referenced in MANIFEST.MF and keep them in dict.
 121         :return:
 122         """
 123         if "package_content" in self.manifest:
 124             for pc in self.manifest.get("package_content"):
 125                 if pc.get("content-type") == "application/sonata.function_descriptor":
 126                     vnfd_path = os.path.join(
 127                         self.package_content_path,
 128                         make_relative_path(pc.get("name")))
 129                     vnfd = load_yaml(vnfd_path)
 130                     self.vnfds[vnfd.get("vnf_name")] = vnfd
 131                     LOG.debug("Loaded VNFD: %r" % vnfd.get("vnf_name"))
 132
 133     def _load_docker_files(self):
 134         """
 135         Get all paths to Dockerfiles from MANIFEST.MF and store them in dict.
 136         :return:
 137         """
 138         if "package_content" in self.manifest:
 139             for df in self.manifest.get("package_content"):
 140                 if df.get("content-type") == "application/sonata.docker_files":
 141                     docker_path = os.path.join(
 142                         self.package_content_path,
 143                         make_relative_path(df.get("name")))
 144                     # FIXME: Mapping to docker image names is hardcoded because of the missing mapping in the example package
 145                     self.local_docker_files[helper_map_docker_name(df.get("name"))] = docker_path
 146                     LOG.debug("Found Dockerfile: %r" % docker_path)
 147
 148     def _build_images_from_dockerfiles(self):
 149         """
 150         Build Docker images for each local Dockerfile found in the package: self.local_docker_files
 151         """
 152         dc = DockerClient()
 153         LOG.info("Building %d Docker images (this may take several minutes) ..." % len(self.local_docker_files))
 154         for k, v in self.local_docker_files.iteritems():
 155             for line in dc.build(path=v.replace("Dockerfile", ""), tag=k, rm=False, nocache=False):
 156                 LOG.debug("DOCKER BUILD: %s" % line)
 157             LOG.info("Docker image created: %s" % k)
 158
 159     def _download_predefined_dockerimages(self):
 160         """
 161         If the package contains URLs to pre-build Docker images, we download them with this method.
 162         """
 163         # TODO implement
 164         pass
 165
 166
 167 """
 168 Resource definitions and API endpoints
 169 """
 170
 171
 172 class Packages(fr.Resource):
 173
 174     def post(self):
 175         """
 176         Upload a *.son service package to the dummy gatekeeper.
 177
 178         We expect request with a *.son file and store it in UPLOAD_FOLDER
 179         :return: UUID
 180         """
 181         try:
 182             # get file contents
 183             son_file = request.files['file']
 184             # generate a uuid to reference this package
 185             service_uuid = str(uuid.uuid4())
 186             file_hash = hashlib.sha1(str(son_file)).hexdigest()
 187             # ensure that upload folder exists
 188             ensure_dir(UPLOAD_FOLDER)
 189             upload_path = os.path.join(UPLOAD_FOLDER, "%s.son" % service_uuid)
 190             # store *.son file to disk
 191             son_file.save(upload_path)
 192             size = os.path.getsize(upload_path)
 193             # create a service object and register it
 194             s = Service(service_uuid, file_hash, upload_path)
 195             GK.register_service_package(service_uuid, s)
 196             # generate the JSON result
 197             return {"service_uuid": service_uuid, "size": size, "sha1": file_hash, "error": None}
 198         except Exception as ex:
 199             LOG.exception("Service package upload failed:")
 200             return {"service_uuid": None, "size": 0, "sha1": None, "error": "upload failed"}
 201
 202     def get(self):
 203         """
 204         Return a list of UUID's of uploaded service packages.
 205         :return: dict/list
 206         """
 207         return {"service_uuid_list": list(GK.services.iterkeys())}
 208
 209
 210 class Instantiations(fr.Resource):
 211
 212     def post(self):
 213         """
 214         Instantiate a service specified by its UUID.
 215         Will return a new UUID to identify the running service instance.
 216         :return: UUID
 217         """
 218         # TODO implement method (start real service)
 219         json_data = request.get_json(force=True)
 220         service_uuid = json_data.get("service_uuid")
 221         if service_uuid is not None:
 222             service_instance_uuid = str(uuid.uuid4())
 223             LOG.info("Starting service %r" % service_uuid)
 224             return {"service_instance_uuid": service_instance_uuid}
 225         return None
 226
 227     def get(self):
 228         """
 229         Returns a list of UUIDs containing all running services.
 230         :return: dict / list
 231         """
 232         # TODO implement method
 233         return {"service_instance_uuid_list": list()}
 234
 235
 236 # create a single, global GK object
 237 GK = Gatekeeper()
 238 # setup Flask
 239 app = Flask(__name__)
 240 app.config['MAX_CONTENT_LENGTH'] = 512 * 1024 * 1024  # 512 MB max upload
 241 api = fr.Api(app)
 242 # define endpoints
 243 api.add_resource(Packages, '/api/packages')
 244 api.add_resource(Instantiations, '/api/instantiations')
 245
 246
 247 def start_rest_api(host, port):
 248     # start the Flask server (not the best performance but ok for our use case)
 249     app.run(host=host,
 250             port=port,
 251             debug=True,
 252             use_reloader=False  # this is needed to run Flask in a non-main thread
 253             )
 254
 255
 256 def ensure_dir(name):
 257     if not os.path.exists(name):
 258         os.makedirs(name)
 259
 260
 261 def load_yaml(path):
 262     with open(path, "r") as f:
 263         try:
 264             r = yaml.load(f)
 265         except yaml.YAMLError as exc:
 266             LOG.exception("YAML parse error")
 267             r = dict()
 268     return r
 269
 270
 271 def make_relative_path(path):
 272     if path.startswith("/"):
 273         return path.replace("/", "", 1)
 274     return path
 275
 276
 277 def helper_map_docker_name(name):
 278     """
 279     Quick hack to fix missing dependency in example package.
 280     """
 281     # TODO remove this when package description is fixed
 282     mapping = {
 283         "/docker_files/iperf/Dockerfile": "iperf_docker",
 284         "/docker_files/firewall/Dockerfile": "fw_docker",
 285         "/docker_files/tcpdump/Dockerfile": "tcpdump_docker"
 286     }
 287     return mapping.get(name)
 288
 289
 290 if __name__ == '__main__':
 291     """
 292     Lets allow to run the API in standalone mode.
 293     """
 294     logging.getLogger("werkzeug").setLevel(logging.INFO)
 295     start_rest_api("0.0.0.0", 8000)
 296