blob: 2a994bddc63a94d6f5910aecf20922ffd6970b9f [file] [log] [blame]
peusterme26487b2016-03-08 14:00:21 +01001"""
peusterm79ef6ae2016-07-08 13:53:57 +02002Copyright (c) 2015 SONATA-NFV and Paderborn University
3ALL RIGHTS RESERVED.
4
5Licensed under the Apache License, Version 2.0 (the "License");
6you may not use this file except in compliance with the License.
7You may obtain a copy of the License at
8
9 http://www.apache.org/licenses/LICENSE-2.0
10
11Unless required by applicable law or agreed to in writing, software
12distributed under the License is distributed on an "AS IS" BASIS,
13WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14See the License for the specific language governing permissions and
15limitations under the License.
16
17Neither the name of the SONATA-NFV [, ANY ADDITIONAL AFFILIATION]
18nor the names of its contributors may be used to endorse or promote
19products derived from this software without specific prior written
20permission.
21
22This work has been performed in the framework of the SONATA project,
23funded by the European Commission under Grant number 671517 through
24the Horizon 2020 and 5G-PPP programmes. The authors would like to
25acknowledge the contributions of their colleagues of the SONATA
26partner consortium (www.sonata-nfv.eu).
27"""
28"""
peusterme26487b2016-03-08 14:00:21 +010029This module implements a simple REST API that behaves like SONATA's gatekeeper.
30
31It is only used to support the development of SONATA's SDK tools and to demonstrate
32the year 1 version of the emulator until the integration with WP4's orchestrator is done.
33"""
34
35import logging
36import os
37import uuid
38import hashlib
peusterm786cd542016-03-14 14:12:17 +010039import zipfile
peusterm7ec665d2016-03-14 15:20:44 +010040import yaml
peusterme66edf72016-08-23 11:11:12 +020041import threading
peustermbdfab7e2016-03-14 16:03:30 +010042from docker import Client as DockerClient
peusterme26487b2016-03-08 14:00:21 +010043from flask import Flask, request
44import flask_restful as fr
wtaverni5b23b662016-06-20 12:26:21 +020045from collections import defaultdict
stevenvanrossemdb2f9432016-08-20 00:01:11 +020046import pkg_resources
peusterme26487b2016-03-08 14:00:21 +010047
peusterm398cd3b2016-03-21 15:04:54 +010048logging.basicConfig()
peusterm786cd542016-03-14 14:12:17 +010049LOG = logging.getLogger("sonata-dummy-gatekeeper")
50LOG.setLevel(logging.DEBUG)
peusterme26487b2016-03-08 14:00:21 +010051logging.getLogger("werkzeug").setLevel(logging.WARNING)
52
peusterm92237dc2016-03-21 15:45:58 +010053GK_STORAGE = "/tmp/son-dummy-gk/"
54UPLOAD_FOLDER = os.path.join(GK_STORAGE, "uploads/")
55CATALOG_FOLDER = os.path.join(GK_STORAGE, "catalog/")
peusterme26487b2016-03-08 14:00:21 +010056
peusterm82d406e2016-05-02 20:52:06 +020057# Enable Dockerfile build functionality
58BUILD_DOCKERFILE = False
59
peusterm398cd3b2016-03-21 15:04:54 +010060# flag to indicate that we run without the emulator (only the bare API for integration testing)
61GK_STANDALONE_MODE = False
62
peusterm56356cb2016-05-03 10:43:43 +020063# should a new version of an image be pulled even if its available
wtaverni5b23b662016-06-20 12:26:21 +020064FORCE_PULL = False
peusterme26487b2016-03-08 14:00:21 +010065
stevenvanrossemdb2f9432016-08-20 00:01:11 +020066# Automatically deploy SAPs (endpoints) of the service as new containers
peustermb1cf5372016-08-23 14:02:09 +020067# Attention: This is not a configuration switch but a global variable! Don't change its default value.
stevenvanrossemdb2f9432016-08-20 00:01:11 +020068DEPLOY_SAP = False
69
peusterm76eb8652016-09-06 11:07:16 +020070# flag to indicate if we use bidirectional forwarding rules in the automatic chaining process
71BIDIRECTIONAL_CHAIN = False
72
peusterme26487b2016-03-08 14:00:21 +010073class Gatekeeper(object):
74
75 def __init__(self):
peusterm786cd542016-03-14 14:12:17 +010076 self.services = dict()
peusterm082378b2016-03-16 20:14:22 +010077 self.dcs = dict()
peusterm3444ae42016-03-16 20:46:41 +010078 self.vnf_counter = 0 # used to generate short names for VNFs (Mininet limitation)
peusterm786cd542016-03-14 14:12:17 +010079 LOG.info("Create SONATA dummy gatekeeper.")
peusterme26487b2016-03-08 14:00:21 +010080
peusterm786cd542016-03-14 14:12:17 +010081 def register_service_package(self, service_uuid, service):
82 """
83 register new service package
84 :param service_uuid
85 :param service object
86 """
87 self.services[service_uuid] = service
88 # lets perform all steps needed to onboard the service
89 service.onboard()
90
peusterm3444ae42016-03-16 20:46:41 +010091 def get_next_vnf_name(self):
92 self.vnf_counter += 1
peusterm398cd3b2016-03-21 15:04:54 +010093 return "vnf%d" % self.vnf_counter
peusterm3444ae42016-03-16 20:46:41 +010094
peusterm786cd542016-03-14 14:12:17 +010095
96class Service(object):
97 """
98 This class represents a NS uploaded as a *.son package to the
99 dummy gatekeeper.
100 Can have multiple running instances of this service.
101 """
102
103 def __init__(self,
104 service_uuid,
105 package_file_hash,
106 package_file_path):
107 self.uuid = service_uuid
108 self.package_file_hash = package_file_hash
109 self.package_file_path = package_file_path
110 self.package_content_path = os.path.join(CATALOG_FOLDER, "services/%s" % self.uuid)
peusterm7ec665d2016-03-14 15:20:44 +0100111 self.manifest = None
112 self.nsd = None
113 self.vnfds = dict()
peustermbdfab7e2016-03-14 16:03:30 +0100114 self.local_docker_files = dict()
peusterm82d406e2016-05-02 20:52:06 +0200115 self.remote_docker_image_urls = dict()
peusterm786cd542016-03-14 14:12:17 +0100116 self.instances = dict()
peusterm6b5224d2016-07-20 13:20:31 +0200117 self.vnf_name2docker_name = dict()
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200118 self.sap_identifiers = set()
peusterm6b5224d2016-07-20 13:20:31 +0200119 # lets generate a set of subnet configurations used for e-line chaining setup
120 self.eline_subnets_src = generate_subnet_strings(50, start=200, subnet_size=24, ip=1)
121 self.eline_subnets_dst = generate_subnet_strings(50, start=200, subnet_size=24, ip=2)
peusterme26487b2016-03-08 14:00:21 +0100122
peusterm786cd542016-03-14 14:12:17 +0100123 def onboard(self):
124 """
125 Do all steps to prepare this service to be instantiated
126 :return:
127 """
128 # 1. extract the contents of the package and store them in our catalog
129 self._unpack_service_package()
130 # 2. read in all descriptor files
peusterm7ec665d2016-03-14 15:20:44 +0100131 self._load_package_descriptor()
132 self._load_nsd()
133 self._load_vnfd()
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200134 if DEPLOY_SAP:
135 self._load_saps()
peusterm786cd542016-03-14 14:12:17 +0100136 # 3. prepare container images (e.g. download or build Dockerfile)
peusterm82d406e2016-05-02 20:52:06 +0200137 if BUILD_DOCKERFILE:
138 self._load_docker_files()
139 self._build_images_from_dockerfiles()
140 else:
141 self._load_docker_urls()
142 self._pull_predefined_dockerimages()
peusterm3bb86bf2016-08-15 09:47:57 +0200143 LOG.info("On-boarded service: %r" % self.manifest.get("name"))
peusterm7ec665d2016-03-14 15:20:44 +0100144
peusterm082378b2016-03-16 20:14:22 +0100145 def start_service(self):
peusterm3444ae42016-03-16 20:46:41 +0100146 """
147 This methods creates and starts a new service instance.
148 It computes placements, iterates over all VNFDs, and starts
149 each VNFD as a Docker container in the data center selected
150 by the placement algorithm.
151 :return:
152 """
153 LOG.info("Starting service %r" % self.uuid)
stevenvanrossemd87fe472016-05-11 11:34:34 +0200154
peusterm3444ae42016-03-16 20:46:41 +0100155 # 1. each service instance gets a new uuid to identify it
peusterm082378b2016-03-16 20:14:22 +0100156 instance_uuid = str(uuid.uuid4())
peusterm3444ae42016-03-16 20:46:41 +0100157 # build a instances dict (a bit like a NSR :))
158 self.instances[instance_uuid] = dict()
159 self.instances[instance_uuid]["vnf_instances"] = list()
stevenvanrossemd87fe472016-05-11 11:34:34 +0200160
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200161 # 2. Configure the chaining of the network functions (currently only E-Line and E-LAN links supported)
162 vnf_id2vnf_name = defaultdict(lambda: "NotExistingNode",
163 reduce(lambda x, y: dict(x, **y),
164 map(lambda d: {d["vnf_id"]: d["vnf_name"]},
165 self.nsd["network_functions"])))
166
167 # 3. compute placement of this service instance (adds DC names to VNFDs)
peusterm398cd3b2016-03-21 15:04:54 +0100168 if not GK_STANDALONE_MODE:
peustermf6459542016-08-31 19:00:17 +0200169 #self._calculate_placement(FirstDcPlacement)
170 self._calculate_placement(RoundRobinDcPlacement)
peusterm3444ae42016-03-16 20:46:41 +0100171 # iterate over all vnfds that we have to start
peusterm082378b2016-03-16 20:14:22 +0100172 for vnfd in self.vnfds.itervalues():
peusterm398cd3b2016-03-21 15:04:54 +0100173 vnfi = None
174 if not GK_STANDALONE_MODE:
175 vnfi = self._start_vnfd(vnfd)
176 self.instances[instance_uuid]["vnf_instances"].append(vnfi)
stevenvanrossemd87fe472016-05-11 11:34:34 +0200177
stevenvanrossemd87fe472016-05-11 11:34:34 +0200178 vlinks = self.nsd["virtual_links"]
179 fwd_links = self.nsd["forwarding_graphs"][0]["constituent_virtual_links"]
180 eline_fwd_links = [l for l in vlinks if (l["id"] in fwd_links) and (l["connectivity_type"] == "E-Line")]
stevenvanrossem6d5019a2016-08-12 23:00:22 +0200181 elan_fwd_links = [l for l in vlinks if (l["id"] in fwd_links) and (l["connectivity_type"] == "E-LAN")]
stevenvanrossemd87fe472016-05-11 11:34:34 +0200182
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200183 # 4a. deploy E-Line links
stevenvanrossemaa6d3a72016-08-10 13:23:24 +0200184 # cookie is used as identifier for the flowrules installed by the dummygatekeeper
185 # eg. different services get a unique cookie for their flowrules
186 cookie = 1
stevenvanrossemd87fe472016-05-11 11:34:34 +0200187 for link in eline_fwd_links:
peusterm6b5224d2016-07-20 13:20:31 +0200188 src_id, src_if_name = link["connection_points_reference"][0].split(":")
189 dst_id, dst_if_name = link["connection_points_reference"][1].split(":")
stevenvanrossemd87fe472016-05-11 11:34:34 +0200190
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200191 # check if there is a SAP in the link
192 if src_id in self.sap_identifiers:
193 src_docker_name = "{0}_{1}".format(src_id, src_if_name)
194 src_id = src_docker_name
195 else:
196 src_docker_name = src_id
197
198 if dst_id in self.sap_identifiers:
199 dst_docker_name = "{0}_{1}".format(dst_id, dst_if_name)
200 dst_id = dst_docker_name
201 else:
202 dst_docker_name = dst_id
203
peusterm6b5224d2016-07-20 13:20:31 +0200204 src_name = vnf_id2vnf_name[src_id]
205 dst_name = vnf_id2vnf_name[dst_id]
peusterm9fb74ec2016-06-16 11:30:55 +0200206
peusterm6b5224d2016-07-20 13:20:31 +0200207 LOG.debug(
208 "Setting up E-Line link. %s(%s:%s) -> %s(%s:%s)" % (
209 src_name, src_id, src_if_name, dst_name, dst_id, dst_if_name))
210
211 if (src_name in self.vnfds) and (dst_name in self.vnfds):
212 network = self.vnfds[src_name].get("dc").net # there should be a cleaner way to find the DCNetwork
peusterm6b5224d2016-07-20 13:20:31 +0200213 LOG.debug(src_docker_name)
214 ret = network.setChain(
215 src_docker_name, dst_docker_name,
216 vnf_src_interface=src_if_name, vnf_dst_interface=dst_if_name,
peusterm76eb8652016-09-06 11:07:16 +0200217 bidirectional=BIDIRECTIONAL_CHAIN, cmd="add-flow", cookie=cookie, priority=10)
stevenvanrossemd87fe472016-05-11 11:34:34 +0200218
peusterm6b5224d2016-07-20 13:20:31 +0200219 # re-configure the VNFs IP assignment and ensure that a new subnet is used for each E-Link
220 src_vnfi = self._get_vnf_instance(instance_uuid, src_name)
221 if src_vnfi is not None:
222 self._vnf_reconfigure_network(src_vnfi, src_if_name, self.eline_subnets_src.pop(0))
223 dst_vnfi = self._get_vnf_instance(instance_uuid, dst_name)
224 if dst_vnfi is not None:
225 self._vnf_reconfigure_network(dst_vnfi, dst_if_name, self.eline_subnets_dst.pop(0))
226
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200227 # 4b. deploy E-LAN links
stevenvanrossem6d5019a2016-08-12 23:00:22 +0200228 base = 10
229 for link in elan_fwd_links:
230 # generate lan ip address
231 ip = 1
232 for intf in link["connection_points_reference"]:
233 ip_address = generate_lan_string("10.0", base, subnet_size=24, ip=ip)
234 vnf_id, intf_name = intf.split(":")
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200235 if vnf_id in self.sap_identifiers:
236 src_docker_name = "{0}_{1}".format(vnf_id, intf_name)
237 vnf_id = src_docker_name
stevenvanrossem6d5019a2016-08-12 23:00:22 +0200238 vnf_name = vnf_id2vnf_name[vnf_id]
239 LOG.debug(
240 "Setting up E-LAN link. %s(%s:%s) -> %s" % (
241 vnf_name, vnf_id, intf_name, ip_address))
242
243 if vnf_name in self.vnfds:
244 # re-configure the VNFs IP assignment and ensure that a new subnet is used for each E-LAN
peustermb1cf5372016-08-23 14:02:09 +0200245 # E-LAN relies on the learning switch capability of Ryu which has to be turned on in the topology
246 # (DCNetwork(controller=RemoteController, enable_learning=True)), so no explicit chaining is necessary.
stevenvanrossem6d5019a2016-08-12 23:00:22 +0200247 vnfi = self._get_vnf_instance(instance_uuid, vnf_name)
248 if vnfi is not None:
249 self._vnf_reconfigure_network(vnfi, intf_name, ip_address)
250 # increase for the next ip address on this E-LAN
251 ip += 1
252 # increase the base ip address for the next E-LAN
253 base += 1
254
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200255 # 5. run the emulator specific entrypoint scripts in the VNFIs of this service instance
peusterm8484b902016-06-21 09:03:35 +0200256 self._trigger_emulator_start_scripts_in_vnfis(self.instances[instance_uuid]["vnf_instances"])
257
peusterm3444ae42016-03-16 20:46:41 +0100258 LOG.info("Service started. Instance id: %r" % instance_uuid)
peusterm082378b2016-03-16 20:14:22 +0100259 return instance_uuid
260
edmaasd454d542016-09-29 13:19:22 +0200261 def stop_service(self):
262 """
263 This method stops a running service instance.
264 It iterates over all VNFDs, stopping them each
265 and removing them from their data center.
266
267 :return:
268 """
269
270
271
272
peusterm398cd3b2016-03-21 15:04:54 +0100273 def _start_vnfd(self, vnfd):
274 """
275 Start a single VNFD of this service
276 :param vnfd: vnfd descriptor dict
277 :return:
278 """
279 # iterate over all deployment units within each VNFDs
280 for u in vnfd.get("virtual_deployment_units"):
281 # 1. get the name of the docker image to start and the assigned DC
peusterm56356cb2016-05-03 10:43:43 +0200282 vnf_name = vnfd.get("name")
283 if vnf_name not in self.remote_docker_image_urls:
284 raise Exception("No image name for %r found. Abort." % vnf_name)
285 docker_name = self.remote_docker_image_urls.get(vnf_name)
peusterm398cd3b2016-03-21 15:04:54 +0100286 target_dc = vnfd.get("dc")
287 # 2. perform some checks to ensure we can start the container
288 assert(docker_name is not None)
289 assert(target_dc is not None)
290 if not self._check_docker_image_exists(docker_name):
291 raise Exception("Docker image %r not found. Abort." % docker_name)
292 # 3. do the dc.startCompute(name="foobar") call to run the container
293 # TODO consider flavors, and other annotations
stevenvanrossemd87fe472016-05-11 11:34:34 +0200294 intfs = vnfd.get("connection_points")
stevenvanrossemeae73082016-08-05 16:22:12 +0200295
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200296 # TODO: get all vnf id's from the nsd for this vnfd and use those as dockername
stevenvanrossem11a021f2016-08-05 13:43:00 +0200297 # use the vnf_id in the nsd as docker name
298 # so deployed containers can be easily mapped back to the nsd
299 vnf_name2id = defaultdict(lambda: "NotExistingNode",
300 reduce(lambda x, y: dict(x, **y),
301 map(lambda d: {d["vnf_name"]: d["vnf_id"]},
302 self.nsd["network_functions"])))
303 self.vnf_name2docker_name[vnf_name] = vnf_name2id[vnf_name]
304 # self.vnf_name2docker_name[vnf_name] = GK.get_next_vnf_name()
305
peusterm6b5224d2016-07-20 13:20:31 +0200306 LOG.info("Starting %r as %r in DC %r" % (vnf_name, self.vnf_name2docker_name[vnf_name], vnfd.get("dc")))
peusterm761c14d2016-07-19 09:31:19 +0200307 LOG.debug("Interfaces for %r: %r" % (vnf_name, intfs))
peusterm6b5224d2016-07-20 13:20:31 +0200308 vnfi = target_dc.startCompute(self.vnf_name2docker_name[vnf_name], network=intfs, image=docker_name, flavor_name="small")
peusterm398cd3b2016-03-21 15:04:54 +0100309 return vnfi
310
edmaasd454d542016-09-29 13:19:22 +0200311 def _stop_vnfd(self, vnf_name):
312 """
313 Stop a VNFD specified by its name.
314
315 :param vnf_name: Name of the vnf to be stopped
316 :return:
317 """
318 if vnf_name not in self.vnfds:
319 raise Exception("VNFD with name %s not found." % vnf_name)
320 vnfd = self.vnfds[vnf_name]
321 dc = vnfd.get("dc")
322 LOG.info("Stopping %r contained in %r in DC %r" % (vnf_name, self.vnf_name2docker_name[vnf_name], dc)
323 dc.stopCompute(self.vnf_name2docker_name[vnf_name])
324
peusterm6b5224d2016-07-20 13:20:31 +0200325 def _get_vnf_instance(self, instance_uuid, name):
326 """
327 Returns the Docker object for the given VNF name (or Docker name).
328 :param instance_uuid: UUID of the service instance to search in.
329 :param name: VNF name or Docker name. We are fuzzy here.
330 :return:
331 """
332 dn = name
333 if name in self.vnf_name2docker_name:
334 dn = self.vnf_name2docker_name[name]
335 for vnfi in self.instances[instance_uuid]["vnf_instances"]:
336 if vnfi.name == dn:
337 return vnfi
338 LOG.warning("No container with name: %r found.")
339 return None
340
341 @staticmethod
342 def _vnf_reconfigure_network(vnfi, if_name, net_str):
343 """
344 Reconfigure the network configuration of a specific interface
345 of a running container.
346 :param vnfi: container instacne
347 :param if_name: interface name
348 :param net_str: network configuration string, e.g., 1.2.3.4/24
349 :return:
350 """
351 intf = vnfi.intf(intf=if_name)
352 if intf is not None:
353 intf.setIP(net_str)
354 LOG.debug("Reconfigured network of %s:%s to %r" % (vnfi.name, if_name, net_str))
355 else:
356 LOG.warning("Interface not found: %s:%s. Network reconfiguration skipped." % (vnfi.name, if_name))
357
358
peusterm8484b902016-06-21 09:03:35 +0200359 def _trigger_emulator_start_scripts_in_vnfis(self, vnfi_list):
360 for vnfi in vnfi_list:
361 config = vnfi.dcinfo.get("Config", dict())
362 env = config.get("Env", list())
363 for env_var in env:
364 if "SON_EMU_CMD=" in env_var:
365 cmd = str(env_var.split("=")[1])
peusterme66edf72016-08-23 11:11:12 +0200366 LOG.info("Executing entry point script in %r: %r" % (vnfi.name, cmd))
367 # execute command in new thread to ensure that GK is not blocked by VNF
368 t = threading.Thread(target=vnfi.cmdPrint, args=(cmd,))
369 t.daemon = True
370 t.start()
peusterm8484b902016-06-21 09:03:35 +0200371
peusterm786cd542016-03-14 14:12:17 +0100372 def _unpack_service_package(self):
373 """
374 unzip *.son file and store contents in CATALOG_FOLDER/services/<service_uuid>/
375 """
peusterm82d406e2016-05-02 20:52:06 +0200376 LOG.info("Unzipping: %r" % self.package_file_path)
peusterm786cd542016-03-14 14:12:17 +0100377 with zipfile.ZipFile(self.package_file_path, "r") as z:
378 z.extractall(self.package_content_path)
379
peusterm82d406e2016-05-02 20:52:06 +0200380
peusterm7ec665d2016-03-14 15:20:44 +0100381 def _load_package_descriptor(self):
382 """
383 Load the main package descriptor YAML and keep it as dict.
384 :return:
385 """
386 self.manifest = load_yaml(
387 os.path.join(
388 self.package_content_path, "META-INF/MANIFEST.MF"))
389
390 def _load_nsd(self):
391 """
392 Load the entry NSD YAML and keep it as dict.
393 :return:
394 """
395 if "entry_service_template" in self.manifest:
396 nsd_path = os.path.join(
397 self.package_content_path,
398 make_relative_path(self.manifest.get("entry_service_template")))
399 self.nsd = load_yaml(nsd_path)
peusterm757fe9a2016-04-04 14:11:58 +0200400 LOG.debug("Loaded NSD: %r" % self.nsd.get("name"))
peusterm7ec665d2016-03-14 15:20:44 +0100401
402 def _load_vnfd(self):
403 """
404 Load all VNFD YAML files referenced in MANIFEST.MF and keep them in dict.
405 :return:
406 """
407 if "package_content" in self.manifest:
408 for pc in self.manifest.get("package_content"):
409 if pc.get("content-type") == "application/sonata.function_descriptor":
410 vnfd_path = os.path.join(
411 self.package_content_path,
412 make_relative_path(pc.get("name")))
413 vnfd = load_yaml(vnfd_path)
peusterm757fe9a2016-04-04 14:11:58 +0200414 self.vnfds[vnfd.get("name")] = vnfd
415 LOG.debug("Loaded VNFD: %r" % vnfd.get("name"))
peusterm7ec665d2016-03-14 15:20:44 +0100416
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200417 def _load_saps(self):
418 # Each Service Access Point (connection_point) in the nsd is getting its own container
419 SAPs = [p["id"] for p in self.nsd["connection_points"] if p["type"] == "interface"]
420 for sap in SAPs:
421 # endpoints needed in this service
422 sap_vnf_id, sap_vnf_interface = sap.split(':')
423 # set of the connection_point ids found in the nsd (in the examples this is 'ns')
424 self.sap_identifiers.add(sap_vnf_id)
425
peusterm76eb8652016-09-06 11:07:16 +0200426 sap_docker_name = "%s_%s" % (sap_vnf_id, sap_vnf_interface)
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200427
428 # add SAP to self.vnfds
429 sapfile = pkg_resources.resource_filename(__name__, "sap_vnfd.yml")
430 sap_vnfd = load_yaml(sapfile)
431 sap_vnfd["connection_points"][0]["id"] = sap_vnf_interface
432 sap_vnfd["name"] = sap_docker_name
433 self.vnfds[sap_docker_name] = sap_vnfd
434 # add SAP vnf to list in the NSD so it is deployed later on
435 # each SAP get a unique VNFD and vnf_id in the NSD
436 self.nsd["network_functions"].append({"vnf_id": sap_docker_name, "vnf_name": sap_docker_name})
437 LOG.debug("Loaded SAP: %r" % sap_vnfd.get("name"))
438
peusterm7ec665d2016-03-14 15:20:44 +0100439 def _load_docker_files(self):
440 """
peusterm9d7d4b02016-03-23 19:56:44 +0100441 Get all paths to Dockerfiles from VNFDs and store them in dict.
peusterm7ec665d2016-03-14 15:20:44 +0100442 :return:
443 """
peusterm9d7d4b02016-03-23 19:56:44 +0100444 for k, v in self.vnfds.iteritems():
445 for vu in v.get("virtual_deployment_units"):
446 if vu.get("vm_image_format") == "docker":
447 vm_image = vu.get("vm_image")
peusterm7ec665d2016-03-14 15:20:44 +0100448 docker_path = os.path.join(
449 self.package_content_path,
peusterm9d7d4b02016-03-23 19:56:44 +0100450 make_relative_path(vm_image))
451 self.local_docker_files[k] = docker_path
peusterm56356cb2016-05-03 10:43:43 +0200452 LOG.debug("Found Dockerfile (%r): %r" % (k, docker_path))
peusterm7ec665d2016-03-14 15:20:44 +0100453
peusterm82d406e2016-05-02 20:52:06 +0200454 def _load_docker_urls(self):
455 """
456 Get all URLs to pre-build docker images in some repo.
457 :return:
458 """
459 for k, v in self.vnfds.iteritems():
460 for vu in v.get("virtual_deployment_units"):
461 if vu.get("vm_image_format") == "docker":
peusterm35ba4052016-05-02 21:21:14 +0200462 url = vu.get("vm_image")
463 if url is not None:
464 url = url.replace("http://", "")
465 self.remote_docker_image_urls[k] = url
peusterm56356cb2016-05-03 10:43:43 +0200466 LOG.debug("Found Docker image URL (%r): %r" % (k, self.remote_docker_image_urls[k]))
peusterm82d406e2016-05-02 20:52:06 +0200467
peustermbdfab7e2016-03-14 16:03:30 +0100468 def _build_images_from_dockerfiles(self):
469 """
470 Build Docker images for each local Dockerfile found in the package: self.local_docker_files
471 """
peusterm398cd3b2016-03-21 15:04:54 +0100472 if GK_STANDALONE_MODE:
473 return # do not build anything in standalone mode
peustermbdfab7e2016-03-14 16:03:30 +0100474 dc = DockerClient()
475 LOG.info("Building %d Docker images (this may take several minutes) ..." % len(self.local_docker_files))
476 for k, v in self.local_docker_files.iteritems():
477 for line in dc.build(path=v.replace("Dockerfile", ""), tag=k, rm=False, nocache=False):
478 LOG.debug("DOCKER BUILD: %s" % line)
479 LOG.info("Docker image created: %s" % k)
480
peusterm82d406e2016-05-02 20:52:06 +0200481 def _pull_predefined_dockerimages(self):
peustermbdfab7e2016-03-14 16:03:30 +0100482 """
483 If the package contains URLs to pre-build Docker images, we download them with this method.
484 """
peusterm35ba4052016-05-02 21:21:14 +0200485 dc = DockerClient()
486 for url in self.remote_docker_image_urls.itervalues():
peusterm56356cb2016-05-03 10:43:43 +0200487 if not FORCE_PULL: # only pull if not present (speedup for development)
488 if len(dc.images(name=url)) > 0:
489 LOG.debug("Image %r present. Skipping pull." % url)
490 continue
peusterm35ba4052016-05-02 21:21:14 +0200491 LOG.info("Pulling image: %r" % url)
492 dc.pull(url,
493 insecure_registry=True)
peusterm786cd542016-03-14 14:12:17 +0100494
peusterm3444ae42016-03-16 20:46:41 +0100495 def _check_docker_image_exists(self, image_name):
peusterm3f307142016-03-16 21:02:53 +0100496 """
497 Query the docker service and check if the given image exists
498 :param image_name: name of the docker image
499 :return:
500 """
501 return len(DockerClient().images(image_name)) > 0
peusterm3444ae42016-03-16 20:46:41 +0100502
peusterm082378b2016-03-16 20:14:22 +0100503 def _calculate_placement(self, algorithm):
504 """
505 Do placement by adding the a field "dc" to
506 each VNFD that points to one of our
507 data center objects known to the gatekeeper.
508 """
509 assert(len(self.vnfds) > 0)
510 assert(len(GK.dcs) > 0)
511 # instantiate algorithm an place
512 p = algorithm()
513 p.place(self.nsd, self.vnfds, GK.dcs)
514 LOG.info("Using placement algorithm: %r" % p.__class__.__name__)
515 # lets print the placement result
516 for name, vnfd in self.vnfds.iteritems():
517 LOG.info("Placed VNF %r on DC %r" % (name, str(vnfd.get("dc"))))
518
519
520"""
521Some (simple) placement algorithms
522"""
523
524
525class FirstDcPlacement(object):
526 """
527 Placement: Always use one and the same data center from the GK.dcs dict.
528 """
529 def place(self, nsd, vnfds, dcs):
530 for name, vnfd in vnfds.iteritems():
531 vnfd["dc"] = list(dcs.itervalues())[0]
532
peusterme26487b2016-03-08 14:00:21 +0100533
peustermf6459542016-08-31 19:00:17 +0200534class RoundRobinDcPlacement(object):
535 """
536 Placement: Distribute VNFs across all available DCs in a round robin fashion.
537 """
peustermf6459542016-08-31 19:00:17 +0200538 def place(self, nsd, vnfds, dcs):
539 c = 0
edmaasd454d542016-09-29 13:19:22 +0200540 dcs_list = list(dcs.itervalues())
peustermf6459542016-08-31 19:00:17 +0200541 for name, vnfd in vnfds.iteritems():
542 vnfd["dc"] = dcs_list[c % len(dcs_list)]
543 c += 1 # inc. c to use next DC
544
545
546
547
peusterme26487b2016-03-08 14:00:21 +0100548"""
549Resource definitions and API endpoints
550"""
551
552
553class Packages(fr.Resource):
554
555 def post(self):
556 """
peusterm26455852016-03-08 14:23:53 +0100557 Upload a *.son service package to the dummy gatekeeper.
558
peusterme26487b2016-03-08 14:00:21 +0100559 We expect request with a *.son file and store it in UPLOAD_FOLDER
peusterm26455852016-03-08 14:23:53 +0100560 :return: UUID
peusterme26487b2016-03-08 14:00:21 +0100561 """
562 try:
563 # get file contents
wtavernib8d9ecb2016-03-25 15:18:31 +0100564 print(request.files)
peusterm593ca582016-03-30 19:55:01 +0200565 # lets search for the package in the request
566 if "package" in request.files:
567 son_file = request.files["package"]
568 # elif "file" in request.files:
569 # son_file = request.files["file"]
570 else:
571 return {"service_uuid": None, "size": 0, "sha1": None, "error": "upload failed. file not found."}, 500
peusterme26487b2016-03-08 14:00:21 +0100572 # generate a uuid to reference this package
573 service_uuid = str(uuid.uuid4())
peusterm786cd542016-03-14 14:12:17 +0100574 file_hash = hashlib.sha1(str(son_file)).hexdigest()
peusterme26487b2016-03-08 14:00:21 +0100575 # ensure that upload folder exists
576 ensure_dir(UPLOAD_FOLDER)
577 upload_path = os.path.join(UPLOAD_FOLDER, "%s.son" % service_uuid)
578 # store *.son file to disk
peusterm786cd542016-03-14 14:12:17 +0100579 son_file.save(upload_path)
peusterme26487b2016-03-08 14:00:21 +0100580 size = os.path.getsize(upload_path)
peusterm786cd542016-03-14 14:12:17 +0100581 # create a service object and register it
582 s = Service(service_uuid, file_hash, upload_path)
583 GK.register_service_package(service_uuid, s)
peusterme26487b2016-03-08 14:00:21 +0100584 # generate the JSON result
peusterm938143e2016-09-15 15:39:36 +0200585 return {"service_uuid": service_uuid, "size": size, "sha1": file_hash, "error": None}, 201
peusterme26487b2016-03-08 14:00:21 +0100586 except Exception as ex:
peusterm786cd542016-03-14 14:12:17 +0100587 LOG.exception("Service package upload failed:")
peusterm593ca582016-03-30 19:55:01 +0200588 return {"service_uuid": None, "size": 0, "sha1": None, "error": "upload failed"}, 500
peusterme26487b2016-03-08 14:00:21 +0100589
590 def get(self):
peusterm26455852016-03-08 14:23:53 +0100591 """
592 Return a list of UUID's of uploaded service packages.
593 :return: dict/list
594 """
peusterm075b46a2016-07-20 17:08:00 +0200595 LOG.info("GET /packages")
peusterm786cd542016-03-14 14:12:17 +0100596 return {"service_uuid_list": list(GK.services.iterkeys())}
peusterme26487b2016-03-08 14:00:21 +0100597
598
599class Instantiations(fr.Resource):
600
601 def post(self):
peusterm26455852016-03-08 14:23:53 +0100602 """
603 Instantiate a service specified by its UUID.
604 Will return a new UUID to identify the running service instance.
605 :return: UUID
606 """
peusterm64b45502016-03-16 21:15:14 +0100607 # try to extract the service uuid from the request
peusterm26455852016-03-08 14:23:53 +0100608 json_data = request.get_json(force=True)
peusterm64b45502016-03-16 21:15:14 +0100609 service_uuid = json_data.get("service_uuid")
610
611 # lets be a bit fuzzy here to make testing easier
612 if service_uuid is None and len(GK.services) > 0:
613 # if we don't get a service uuid, we simple start the first service in the list
614 service_uuid = list(GK.services.iterkeys())[0]
615
peustermbea87372016-03-16 19:37:35 +0100616 if service_uuid in GK.services:
peusterm64b45502016-03-16 21:15:14 +0100617 # ok, we have a service uuid, lets start the service
peustermbea87372016-03-16 19:37:35 +0100618 service_instance_uuid = GK.services.get(service_uuid).start_service()
peusterm26455852016-03-08 14:23:53 +0100619 return {"service_instance_uuid": service_instance_uuid}
peustermbea87372016-03-16 19:37:35 +0100620 return "Service not found", 404
peusterme26487b2016-03-08 14:00:21 +0100621
622 def get(self):
peusterm26455852016-03-08 14:23:53 +0100623 """
624 Returns a list of UUIDs containing all running services.
625 :return: dict / list
626 """
peusterm075b46a2016-07-20 17:08:00 +0200627 LOG.info("GET /instantiations")
628 return {"service_instantiations_list": [
peusterm64b45502016-03-16 21:15:14 +0100629 list(s.instances.iterkeys()) for s in GK.services.itervalues()]}
peusterm786cd542016-03-14 14:12:17 +0100630
edmaasd454d542016-09-29 13:19:22 +0200631 def delete(self):
632 """
633 Stops a running service specified by its UUID.
634
635 :return:
636 """
637 # try to extract the service UUID from the request
638 json_data = request.get_json(force=True)
639 service_uuid = json_data.get("service_uuid")
640
641 if service_uuid in GK.services:
642 # valid service UUID, stop service
643 GK.services.get(service_uuid).stop_service()
644 return "", 0
645 return "Service not found", 404
646
peusterme26487b2016-03-08 14:00:21 +0100647
648# create a single, global GK object
649GK = Gatekeeper()
650# setup Flask
651app = Flask(__name__)
652app.config['MAX_CONTENT_LENGTH'] = 512 * 1024 * 1024 # 512 MB max upload
653api = fr.Api(app)
654# define endpoints
peusterm593ca582016-03-30 19:55:01 +0200655api.add_resource(Packages, '/packages')
656api.add_resource(Instantiations, '/instantiations')
peusterme26487b2016-03-08 14:00:21 +0100657
658
peusterm082378b2016-03-16 20:14:22 +0100659def start_rest_api(host, port, datacenters=dict()):
peustermbea87372016-03-16 19:37:35 +0100660 GK.dcs = datacenters
peusterme26487b2016-03-08 14:00:21 +0100661 # start the Flask server (not the best performance but ok for our use case)
662 app.run(host=host,
663 port=port,
664 debug=True,
665 use_reloader=False # this is needed to run Flask in a non-main thread
666 )
667
668
669def ensure_dir(name):
670 if not os.path.exists(name):
peusterm7ec665d2016-03-14 15:20:44 +0100671 os.makedirs(name)
672
673
674def load_yaml(path):
675 with open(path, "r") as f:
676 try:
677 r = yaml.load(f)
678 except yaml.YAMLError as exc:
679 LOG.exception("YAML parse error")
680 r = dict()
681 return r
682
683
684def make_relative_path(path):
peusterm9d7d4b02016-03-23 19:56:44 +0100685 if path.startswith("file://"):
686 path = path.replace("file://", "", 1)
peusterm7ec665d2016-03-14 15:20:44 +0100687 if path.startswith("/"):
peusterm9d7d4b02016-03-23 19:56:44 +0100688 path = path.replace("/", "", 1)
peusterm7ec665d2016-03-14 15:20:44 +0100689 return path
690
691
stevenvanrossem6d5019a2016-08-12 23:00:22 +0200692def generate_lan_string(prefix, base, subnet_size=24, ip=0):
693 """
694 Helper to generate different network configuration strings.
695 """
696 r = "%s.%d.%d/%d" % (prefix, base, ip, subnet_size)
697 return r
698
699
peusterm6b5224d2016-07-20 13:20:31 +0200700def generate_subnet_strings(n, start=1, subnet_size=24, ip=0):
701 """
702 Helper to generate different network configuration strings.
703 """
704 r = list()
705 for i in range(start, start + n):
706 r.append("%d.0.0.%d/%d" % (i, ip, subnet_size))
707 return r
708
709
peusterme26487b2016-03-08 14:00:21 +0100710if __name__ == '__main__':
711 """
712 Lets allow to run the API in standalone mode.
713 """
peusterm398cd3b2016-03-21 15:04:54 +0100714 GK_STANDALONE_MODE = True
peusterme26487b2016-03-08 14:00:21 +0100715 logging.getLogger("werkzeug").setLevel(logging.INFO)
716 start_rest_api("0.0.0.0", 8000)
717