X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=src%2Femuvim%2Fapi%2Fsonata%2Fdummygatekeeper.py;h=68031e494fa71fa4e0ce636a26423ce48ea6d6fb;hb=3444ae45d2b2819993cd972ffe6a572b61fce621;hp=71fd94c6b30abb2c464685698083bfe80edfc3cd;hpb=786cd5487fb74d3e91bdf8b316c2e64e2fc9c575;p=osm%2Fvim-emu.git diff --git a/src/emuvim/api/sonata/dummygatekeeper.py b/src/emuvim/api/sonata/dummygatekeeper.py index 71fd94c..68031e4 100644 --- a/src/emuvim/api/sonata/dummygatekeeper.py +++ b/src/emuvim/api/sonata/dummygatekeeper.py @@ -10,6 +10,8 @@ import os import uuid import hashlib import zipfile +import yaml +from docker import Client as DockerClient from flask import Flask, request import flask_restful as fr @@ -26,6 +28,8 @@ class Gatekeeper(object): def __init__(self): self.services = dict() + self.dcs = dict() + self.vnf_counter = 0 # used to generate short names for VNFs (Mininet limitation) LOG.info("Create SONATA dummy gatekeeper.") def register_service_package(self, service_uuid, service): @@ -38,6 +42,10 @@ class Gatekeeper(object): # lets perform all steps needed to onboard the service service.onboard() + def get_next_vnf_name(self): + self.vnf_counter += 1 + return "sonvnf%d" % self.vnf_counter + class Service(object): """ @@ -54,15 +62,11 @@ class Service(object): self.package_file_hash = package_file_hash self.package_file_path = package_file_path self.package_content_path = os.path.join(CATALOG_FOLDER, "services/%s" % self.uuid) + self.manifest = None + self.nsd = None + self.vnfds = dict() + self.local_docker_files = dict() self.instances = dict() - LOG.info("Created service: %r" % self.uuid) - - def start_service(self, service_uuid): - # TODO implement method - # 1. parse descriptors - # 2. do the corresponding dc.startCompute(name="foobar") calls - # 3. store references to the compute objects in self.instantiations - pass def onboard(self): """ @@ -72,7 +76,51 @@ class Service(object): # 1. extract the contents of the package and store them in our catalog self._unpack_service_package() # 2. read in all descriptor files + self._load_package_descriptor() + self._load_nsd() + self._load_vnfd() + self._load_docker_files() # 3. prepare container images (e.g. download or build Dockerfile) + self._build_images_from_dockerfiles() + self._download_predefined_dockerimages() + + LOG.info("On-boarded service: %r" % self.manifest.get("package_name")) + + def start_service(self): + """ + This methods creates and starts a new service instance. + It computes placements, iterates over all VNFDs, and starts + each VNFD as a Docker container in the data center selected + by the placement algorithm. + :return: + """ + LOG.info("Starting service %r" % self.uuid) + # 1. each service instance gets a new uuid to identify it + instance_uuid = str(uuid.uuid4()) + # build a instances dict (a bit like a NSR :)) + self.instances[instance_uuid] = dict() + self.instances[instance_uuid]["vnf_instances"] = list() + # 2. compute placement of this service instance (adds DC names to VNFDs) + self._calculate_placement(FirstDcPlacement) + # iterate over all vnfds that we have to start + for vnfd in self.vnfds.itervalues(): + # iterate over all deployment units within each VNFDs + for u in vnfd.get("virtual_deployment_units"): + # 3. get the name of the docker image to start and the assigned DC + docker_name = u.get("vm_image") + target_dc = vnfd.get("dc") + # 4. perform some checks to ensure we can start the container + assert(docker_name is not None) + assert(target_dc is not None) + if not self._check_docker_image_exists(docker_name): + raise Exception("Docker image %r not found. Abort." % docker_name) + # 5. do the dc.startCompute(name="foobar") call to run the container + # TODO consider flavors, and other annotations + vnfi = target_dc.startCompute(GK.get_next_vnf_name(), image=docker_name, flavor_name="small") + # 6. store references to the compute objects in self.instances + self.instances[instance_uuid]["vnf_instances"].append(vnfi) + LOG.info("Service started. Instance id: %r" % instance_uuid) + return instance_uuid def _unpack_service_package(self): """ @@ -81,9 +129,108 @@ class Service(object): with zipfile.ZipFile(self.package_file_path, "r") as z: z.extractall(self.package_content_path) - def _build_images_from_dockerfile(self): + def _load_package_descriptor(self): + """ + Load the main package descriptor YAML and keep it as dict. + :return: + """ + self.manifest = load_yaml( + os.path.join( + self.package_content_path, "META-INF/MANIFEST.MF")) + + def _load_nsd(self): + """ + Load the entry NSD YAML and keep it as dict. + :return: + """ + if "entry_service_template" in self.manifest: + nsd_path = os.path.join( + self.package_content_path, + make_relative_path(self.manifest.get("entry_service_template"))) + self.nsd = load_yaml(nsd_path) + LOG.debug("Loaded NSD: %r" % self.nsd.get("ns_name")) + + def _load_vnfd(self): + """ + Load all VNFD YAML files referenced in MANIFEST.MF and keep them in dict. + :return: + """ + if "package_content" in self.manifest: + for pc in self.manifest.get("package_content"): + if pc.get("content-type") == "application/sonata.function_descriptor": + vnfd_path = os.path.join( + self.package_content_path, + make_relative_path(pc.get("name"))) + vnfd = load_yaml(vnfd_path) + self.vnfds[vnfd.get("vnf_name")] = vnfd + LOG.debug("Loaded VNFD: %r" % vnfd.get("vnf_name")) + + def _load_docker_files(self): + """ + Get all paths to Dockerfiles from MANIFEST.MF and store them in dict. + :return: + """ + if "package_content" in self.manifest: + for df in self.manifest.get("package_content"): + if df.get("content-type") == "application/sonata.docker_files": + docker_path = os.path.join( + self.package_content_path, + make_relative_path(df.get("name"))) + # FIXME: Mapping to docker image names is hardcoded because of the missing mapping in the example package + self.local_docker_files[helper_map_docker_name(df.get("name"))] = docker_path + LOG.debug("Found Dockerfile: %r" % docker_path) + + def _build_images_from_dockerfiles(self): + """ + Build Docker images for each local Dockerfile found in the package: self.local_docker_files + """ + dc = DockerClient() + LOG.info("Building %d Docker images (this may take several minutes) ..." % len(self.local_docker_files)) + for k, v in self.local_docker_files.iteritems(): + for line in dc.build(path=v.replace("Dockerfile", ""), tag=k, rm=False, nocache=False): + LOG.debug("DOCKER BUILD: %s" % line) + LOG.info("Docker image created: %s" % k) + + def _download_predefined_dockerimages(self): + """ + If the package contains URLs to pre-build Docker images, we download them with this method. + """ + # TODO implement pass + + def _check_docker_image_exists(self, image_name): # TODO implement + return True + + def _calculate_placement(self, algorithm): + """ + Do placement by adding the a field "dc" to + each VNFD that points to one of our + data center objects known to the gatekeeper. + """ + assert(len(self.vnfds) > 0) + assert(len(GK.dcs) > 0) + # instantiate algorithm an place + p = algorithm() + p.place(self.nsd, self.vnfds, GK.dcs) + LOG.info("Using placement algorithm: %r" % p.__class__.__name__) + # lets print the placement result + for name, vnfd in self.vnfds.iteritems(): + LOG.info("Placed VNF %r on DC %r" % (name, str(vnfd.get("dc")))) + + +""" +Some (simple) placement algorithms +""" + + +class FirstDcPlacement(object): + """ + Placement: Always use one and the same data center from the GK.dcs dict. + """ + def place(self, nsd, vnfds, dcs): + for name, vnfd in vnfds.iteritems(): + vnfd["dc"] = list(dcs.itervalues())[0] """ @@ -139,12 +286,11 @@ class Instantiations(fr.Resource): """ # TODO implement method (start real service) json_data = request.get_json(force=True) - service_uuid = json_data.get("service_uuid") - if service_uuid is not None: - service_instance_uuid = str(uuid.uuid4()) - LOG.info("Starting service %r" % service_uuid) + service_uuid = list(GK.services.iterkeys())[0] #json_data.get("service_uuid") # TODO only for quick testing + if service_uuid in GK.services: + service_instance_uuid = GK.services.get(service_uuid).start_service() return {"service_instance_uuid": service_instance_uuid} - return None + return "Service not found", 404 def get(self): """ @@ -166,7 +312,8 @@ api.add_resource(Packages, '/api/packages') api.add_resource(Instantiations, '/api/instantiations') -def start_rest_api(host, port): +def start_rest_api(host, port, datacenters=dict()): + GK.dcs = datacenters # start the Flask server (not the best performance but ok for our use case) app.run(host=host, port=port, @@ -177,7 +324,36 @@ def start_rest_api(host, port): def ensure_dir(name): if not os.path.exists(name): - os.makedirs(name) + os.makedirs(name) + + +def load_yaml(path): + with open(path, "r") as f: + try: + r = yaml.load(f) + except yaml.YAMLError as exc: + LOG.exception("YAML parse error") + r = dict() + return r + + +def make_relative_path(path): + if path.startswith("/"): + return path.replace("/", "", 1) + return path + + +def helper_map_docker_name(name): + """ + Quick hack to fix missing dependency in example package. + """ + # TODO remove this when package description is fixed + mapping = { + "/docker_files/iperf/Dockerfile": "iperf_docker", + "/docker_files/firewall/Dockerfile": "fw_docker", + "/docker_files/tcpdump/Dockerfile": "tcpdump_docker" + } + return mapping.get(name) if __name__ == '__main__':