blob: 00201f9a9dd096dc10ed17bebedaed5dbbf498ac [file] [log] [blame]
peusterme26487b2016-03-08 14:00:21 +01001"""
peusterm79ef6ae2016-07-08 13:53:57 +02002Copyright (c) 2015 SONATA-NFV and Paderborn University
3ALL RIGHTS RESERVED.
4
5Licensed under the Apache License, Version 2.0 (the "License");
6you may not use this file except in compliance with the License.
7You may obtain a copy of the License at
8
9 http://www.apache.org/licenses/LICENSE-2.0
10
11Unless required by applicable law or agreed to in writing, software
12distributed under the License is distributed on an "AS IS" BASIS,
13WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14See the License for the specific language governing permissions and
15limitations under the License.
16
17Neither the name of the SONATA-NFV [, ANY ADDITIONAL AFFILIATION]
18nor the names of its contributors may be used to endorse or promote
19products derived from this software without specific prior written
20permission.
21
22This work has been performed in the framework of the SONATA project,
23funded by the European Commission under Grant number 671517 through
24the Horizon 2020 and 5G-PPP programmes. The authors would like to
25acknowledge the contributions of their colleagues of the SONATA
26partner consortium (www.sonata-nfv.eu).
27"""
28"""
peusterme26487b2016-03-08 14:00:21 +010029This module implements a simple REST API that behaves like SONATA's gatekeeper.
30
31It is only used to support the development of SONATA's SDK tools and to demonstrate
32the year 1 version of the emulator until the integration with WP4's orchestrator is done.
33"""
34
35import logging
36import os
37import uuid
38import hashlib
peusterm786cd542016-03-14 14:12:17 +010039import zipfile
peusterm7ec665d2016-03-14 15:20:44 +010040import yaml
peusterme66edf72016-08-23 11:11:12 +020041import threading
peustermbdfab7e2016-03-14 16:03:30 +010042from docker import Client as DockerClient
peusterme26487b2016-03-08 14:00:21 +010043from flask import Flask, request
44import flask_restful as fr
wtaverni5b23b662016-06-20 12:26:21 +020045from collections import defaultdict
stevenvanrossemdb2f9432016-08-20 00:01:11 +020046import pkg_resources
peusterme26487b2016-03-08 14:00:21 +010047
peusterm398cd3b2016-03-21 15:04:54 +010048logging.basicConfig()
peusterm786cd542016-03-14 14:12:17 +010049LOG = logging.getLogger("sonata-dummy-gatekeeper")
50LOG.setLevel(logging.DEBUG)
peusterme26487b2016-03-08 14:00:21 +010051logging.getLogger("werkzeug").setLevel(logging.WARNING)
52
peusterm92237dc2016-03-21 15:45:58 +010053GK_STORAGE = "/tmp/son-dummy-gk/"
54UPLOAD_FOLDER = os.path.join(GK_STORAGE, "uploads/")
55CATALOG_FOLDER = os.path.join(GK_STORAGE, "catalog/")
peusterme26487b2016-03-08 14:00:21 +010056
peusterm82d406e2016-05-02 20:52:06 +020057# Enable Dockerfile build functionality
58BUILD_DOCKERFILE = False
59
peusterm398cd3b2016-03-21 15:04:54 +010060# flag to indicate that we run without the emulator (only the bare API for integration testing)
61GK_STANDALONE_MODE = False
62
peusterm56356cb2016-05-03 10:43:43 +020063# should a new version of an image be pulled even if its available
wtaverni5b23b662016-06-20 12:26:21 +020064FORCE_PULL = False
peusterme26487b2016-03-08 14:00:21 +010065
stevenvanrossemdb2f9432016-08-20 00:01:11 +020066# Automatically deploy SAPs (endpoints) of the service as new containers
peustermb1cf5372016-08-23 14:02:09 +020067# Attention: This is not a configuration switch but a global variable! Don't change its default value.
stevenvanrossemdb2f9432016-08-20 00:01:11 +020068DEPLOY_SAP = False
69
peusterm76eb8652016-09-06 11:07:16 +020070# flag to indicate if we use bidirectional forwarding rules in the automatic chaining process
71BIDIRECTIONAL_CHAIN = False
72
peusterme26487b2016-03-08 14:00:21 +010073class Gatekeeper(object):
74
75 def __init__(self):
peusterm786cd542016-03-14 14:12:17 +010076 self.services = dict()
peusterm082378b2016-03-16 20:14:22 +010077 self.dcs = dict()
peusterm3444ae42016-03-16 20:46:41 +010078 self.vnf_counter = 0 # used to generate short names for VNFs (Mininet limitation)
peusterm786cd542016-03-14 14:12:17 +010079 LOG.info("Create SONATA dummy gatekeeper.")
peusterme26487b2016-03-08 14:00:21 +010080
peusterm786cd542016-03-14 14:12:17 +010081 def register_service_package(self, service_uuid, service):
82 """
83 register new service package
84 :param service_uuid
85 :param service object
86 """
87 self.services[service_uuid] = service
88 # lets perform all steps needed to onboard the service
89 service.onboard()
90
peusterm3444ae42016-03-16 20:46:41 +010091 def get_next_vnf_name(self):
92 self.vnf_counter += 1
peusterm398cd3b2016-03-21 15:04:54 +010093 return "vnf%d" % self.vnf_counter
peusterm3444ae42016-03-16 20:46:41 +010094
peusterm786cd542016-03-14 14:12:17 +010095
96class Service(object):
97 """
98 This class represents a NS uploaded as a *.son package to the
99 dummy gatekeeper.
100 Can have multiple running instances of this service.
101 """
102
103 def __init__(self,
104 service_uuid,
105 package_file_hash,
106 package_file_path):
107 self.uuid = service_uuid
108 self.package_file_hash = package_file_hash
109 self.package_file_path = package_file_path
110 self.package_content_path = os.path.join(CATALOG_FOLDER, "services/%s" % self.uuid)
peusterm7ec665d2016-03-14 15:20:44 +0100111 self.manifest = None
112 self.nsd = None
113 self.vnfds = dict()
peustermbdfab7e2016-03-14 16:03:30 +0100114 self.local_docker_files = dict()
peusterm82d406e2016-05-02 20:52:06 +0200115 self.remote_docker_image_urls = dict()
peusterm786cd542016-03-14 14:12:17 +0100116 self.instances = dict()
peusterm6b5224d2016-07-20 13:20:31 +0200117 self.vnf_name2docker_name = dict()
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200118 self.sap_identifiers = set()
peusterm6b5224d2016-07-20 13:20:31 +0200119 # lets generate a set of subnet configurations used for e-line chaining setup
120 self.eline_subnets_src = generate_subnet_strings(50, start=200, subnet_size=24, ip=1)
121 self.eline_subnets_dst = generate_subnet_strings(50, start=200, subnet_size=24, ip=2)
peusterme26487b2016-03-08 14:00:21 +0100122
peusterm786cd542016-03-14 14:12:17 +0100123 def onboard(self):
124 """
125 Do all steps to prepare this service to be instantiated
126 :return:
127 """
128 # 1. extract the contents of the package and store them in our catalog
129 self._unpack_service_package()
130 # 2. read in all descriptor files
peusterm7ec665d2016-03-14 15:20:44 +0100131 self._load_package_descriptor()
132 self._load_nsd()
133 self._load_vnfd()
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200134 if DEPLOY_SAP:
135 self._load_saps()
peusterm786cd542016-03-14 14:12:17 +0100136 # 3. prepare container images (e.g. download or build Dockerfile)
peusterm82d406e2016-05-02 20:52:06 +0200137 if BUILD_DOCKERFILE:
138 self._load_docker_files()
139 self._build_images_from_dockerfiles()
140 else:
141 self._load_docker_urls()
142 self._pull_predefined_dockerimages()
peusterm3bb86bf2016-08-15 09:47:57 +0200143 LOG.info("On-boarded service: %r" % self.manifest.get("name"))
peusterm7ec665d2016-03-14 15:20:44 +0100144
peusterm082378b2016-03-16 20:14:22 +0100145 def start_service(self):
peusterm3444ae42016-03-16 20:46:41 +0100146 """
147 This methods creates and starts a new service instance.
148 It computes placements, iterates over all VNFDs, and starts
149 each VNFD as a Docker container in the data center selected
150 by the placement algorithm.
151 :return:
152 """
153 LOG.info("Starting service %r" % self.uuid)
stevenvanrossemd87fe472016-05-11 11:34:34 +0200154
peusterm3444ae42016-03-16 20:46:41 +0100155 # 1. each service instance gets a new uuid to identify it
peusterm082378b2016-03-16 20:14:22 +0100156 instance_uuid = str(uuid.uuid4())
peusterm3444ae42016-03-16 20:46:41 +0100157 # build a instances dict (a bit like a NSR :))
158 self.instances[instance_uuid] = dict()
159 self.instances[instance_uuid]["vnf_instances"] = list()
stevenvanrossemd87fe472016-05-11 11:34:34 +0200160
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200161 # 2. Configure the chaining of the network functions (currently only E-Line and E-LAN links supported)
162 vnf_id2vnf_name = defaultdict(lambda: "NotExistingNode",
163 reduce(lambda x, y: dict(x, **y),
164 map(lambda d: {d["vnf_id"]: d["vnf_name"]},
165 self.nsd["network_functions"])))
166
167 # 3. compute placement of this service instance (adds DC names to VNFDs)
peusterm398cd3b2016-03-21 15:04:54 +0100168 if not GK_STANDALONE_MODE:
peustermf6459542016-08-31 19:00:17 +0200169 #self._calculate_placement(FirstDcPlacement)
170 self._calculate_placement(RoundRobinDcPlacement)
peusterm3444ae42016-03-16 20:46:41 +0100171 # iterate over all vnfds that we have to start
peusterm082378b2016-03-16 20:14:22 +0100172 for vnfd in self.vnfds.itervalues():
peusterm398cd3b2016-03-21 15:04:54 +0100173 vnfi = None
174 if not GK_STANDALONE_MODE:
175 vnfi = self._start_vnfd(vnfd)
176 self.instances[instance_uuid]["vnf_instances"].append(vnfi)
stevenvanrossemd87fe472016-05-11 11:34:34 +0200177
stevenvanrossemd87fe472016-05-11 11:34:34 +0200178 vlinks = self.nsd["virtual_links"]
179 fwd_links = self.nsd["forwarding_graphs"][0]["constituent_virtual_links"]
180 eline_fwd_links = [l for l in vlinks if (l["id"] in fwd_links) and (l["connectivity_type"] == "E-Line")]
stevenvanrossem6d5019a2016-08-12 23:00:22 +0200181 elan_fwd_links = [l for l in vlinks if (l["id"] in fwd_links) and (l["connectivity_type"] == "E-LAN")]
stevenvanrossemd87fe472016-05-11 11:34:34 +0200182
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200183 # 4a. deploy E-Line links
stevenvanrossemaa6d3a72016-08-10 13:23:24 +0200184 # cookie is used as identifier for the flowrules installed by the dummygatekeeper
185 # eg. different services get a unique cookie for their flowrules
186 cookie = 1
stevenvanrossemd87fe472016-05-11 11:34:34 +0200187 for link in eline_fwd_links:
peusterm6b5224d2016-07-20 13:20:31 +0200188 src_id, src_if_name = link["connection_points_reference"][0].split(":")
189 dst_id, dst_if_name = link["connection_points_reference"][1].split(":")
stevenvanrossemd87fe472016-05-11 11:34:34 +0200190
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200191 # check if there is a SAP in the link
192 if src_id in self.sap_identifiers:
193 src_docker_name = "{0}_{1}".format(src_id, src_if_name)
194 src_id = src_docker_name
195 else:
196 src_docker_name = src_id
197
198 if dst_id in self.sap_identifiers:
199 dst_docker_name = "{0}_{1}".format(dst_id, dst_if_name)
200 dst_id = dst_docker_name
201 else:
202 dst_docker_name = dst_id
203
peusterm6b5224d2016-07-20 13:20:31 +0200204 src_name = vnf_id2vnf_name[src_id]
205 dst_name = vnf_id2vnf_name[dst_id]
peusterm9fb74ec2016-06-16 11:30:55 +0200206
peusterm6b5224d2016-07-20 13:20:31 +0200207 LOG.debug(
208 "Setting up E-Line link. %s(%s:%s) -> %s(%s:%s)" % (
209 src_name, src_id, src_if_name, dst_name, dst_id, dst_if_name))
210
211 if (src_name in self.vnfds) and (dst_name in self.vnfds):
212 network = self.vnfds[src_name].get("dc").net # there should be a cleaner way to find the DCNetwork
peusterm6b5224d2016-07-20 13:20:31 +0200213 LOG.debug(src_docker_name)
214 ret = network.setChain(
215 src_docker_name, dst_docker_name,
216 vnf_src_interface=src_if_name, vnf_dst_interface=dst_if_name,
peusterm76eb8652016-09-06 11:07:16 +0200217 bidirectional=BIDIRECTIONAL_CHAIN, cmd="add-flow", cookie=cookie, priority=10)
stevenvanrossemd87fe472016-05-11 11:34:34 +0200218
peusterm6b5224d2016-07-20 13:20:31 +0200219 # re-configure the VNFs IP assignment and ensure that a new subnet is used for each E-Link
220 src_vnfi = self._get_vnf_instance(instance_uuid, src_name)
221 if src_vnfi is not None:
222 self._vnf_reconfigure_network(src_vnfi, src_if_name, self.eline_subnets_src.pop(0))
223 dst_vnfi = self._get_vnf_instance(instance_uuid, dst_name)
224 if dst_vnfi is not None:
225 self._vnf_reconfigure_network(dst_vnfi, dst_if_name, self.eline_subnets_dst.pop(0))
226
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200227 # 4b. deploy E-LAN links
stevenvanrossem6d5019a2016-08-12 23:00:22 +0200228 base = 10
229 for link in elan_fwd_links:
stevenvanrossemdc3bfd02016-11-04 15:33:28 +0100230
231 elan=[]
232
stevenvanrossem6d5019a2016-08-12 23:00:22 +0200233 # generate lan ip address
234 ip = 1
235 for intf in link["connection_points_reference"]:
236 ip_address = generate_lan_string("10.0", base, subnet_size=24, ip=ip)
237 vnf_id, intf_name = intf.split(":")
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200238 if vnf_id in self.sap_identifiers:
239 src_docker_name = "{0}_{1}".format(vnf_id, intf_name)
240 vnf_id = src_docker_name
stevenvanrossemdc3bfd02016-11-04 15:33:28 +0100241 else:
242 src_docker_name = vnf_id
stevenvanrossem6d5019a2016-08-12 23:00:22 +0200243 vnf_name = vnf_id2vnf_name[vnf_id]
244 LOG.debug(
245 "Setting up E-LAN link. %s(%s:%s) -> %s" % (
246 vnf_name, vnf_id, intf_name, ip_address))
247
248 if vnf_name in self.vnfds:
249 # re-configure the VNFs IP assignment and ensure that a new subnet is used for each E-LAN
peustermb1cf5372016-08-23 14:02:09 +0200250 # E-LAN relies on the learning switch capability of Ryu which has to be turned on in the topology
251 # (DCNetwork(controller=RemoteController, enable_learning=True)), so no explicit chaining is necessary.
stevenvanrossem6d5019a2016-08-12 23:00:22 +0200252 vnfi = self._get_vnf_instance(instance_uuid, vnf_name)
253 if vnfi is not None:
254 self._vnf_reconfigure_network(vnfi, intf_name, ip_address)
255 # increase for the next ip address on this E-LAN
256 ip += 1
stevenvanrossemdc3bfd02016-11-04 15:33:28 +0100257
258 # add this vnf and interface to the E-LAN for tagging
259 network = self.vnfds[vnf_name].get("dc").net # there should be a cleaner way to find the DCNetwork
260 elan.append({'name':src_docker_name,'interface':intf_name})
261
262
263 # install the VLAN tags for this E-LAN
264 print(elan)
265 network.setLAN(elan)
stevenvanrossem6d5019a2016-08-12 23:00:22 +0200266 # increase the base ip address for the next E-LAN
267 base += 1
268
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200269 # 5. run the emulator specific entrypoint scripts in the VNFIs of this service instance
peusterm8484b902016-06-21 09:03:35 +0200270 self._trigger_emulator_start_scripts_in_vnfis(self.instances[instance_uuid]["vnf_instances"])
271
peusterm3444ae42016-03-16 20:46:41 +0100272 LOG.info("Service started. Instance id: %r" % instance_uuid)
peusterm082378b2016-03-16 20:14:22 +0100273 return instance_uuid
274
peusterm398cd3b2016-03-21 15:04:54 +0100275 def _start_vnfd(self, vnfd):
276 """
277 Start a single VNFD of this service
278 :param vnfd: vnfd descriptor dict
279 :return:
280 """
281 # iterate over all deployment units within each VNFDs
282 for u in vnfd.get("virtual_deployment_units"):
283 # 1. get the name of the docker image to start and the assigned DC
peusterm56356cb2016-05-03 10:43:43 +0200284 vnf_name = vnfd.get("name")
285 if vnf_name not in self.remote_docker_image_urls:
286 raise Exception("No image name for %r found. Abort." % vnf_name)
287 docker_name = self.remote_docker_image_urls.get(vnf_name)
peusterm398cd3b2016-03-21 15:04:54 +0100288 target_dc = vnfd.get("dc")
289 # 2. perform some checks to ensure we can start the container
290 assert(docker_name is not None)
291 assert(target_dc is not None)
292 if not self._check_docker_image_exists(docker_name):
293 raise Exception("Docker image %r not found. Abort." % docker_name)
294 # 3. do the dc.startCompute(name="foobar") call to run the container
295 # TODO consider flavors, and other annotations
stevenvanrossemd87fe472016-05-11 11:34:34 +0200296 intfs = vnfd.get("connection_points")
stevenvanrossemeae73082016-08-05 16:22:12 +0200297
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200298 # TODO: get all vnf id's from the nsd for this vnfd and use those as dockername
stevenvanrossem11a021f2016-08-05 13:43:00 +0200299 # use the vnf_id in the nsd as docker name
300 # so deployed containers can be easily mapped back to the nsd
301 vnf_name2id = defaultdict(lambda: "NotExistingNode",
302 reduce(lambda x, y: dict(x, **y),
303 map(lambda d: {d["vnf_name"]: d["vnf_id"]},
304 self.nsd["network_functions"])))
305 self.vnf_name2docker_name[vnf_name] = vnf_name2id[vnf_name]
306 # self.vnf_name2docker_name[vnf_name] = GK.get_next_vnf_name()
307
peusterm6b5224d2016-07-20 13:20:31 +0200308 LOG.info("Starting %r as %r in DC %r" % (vnf_name, self.vnf_name2docker_name[vnf_name], vnfd.get("dc")))
peusterm761c14d2016-07-19 09:31:19 +0200309 LOG.debug("Interfaces for %r: %r" % (vnf_name, intfs))
peusterm6b5224d2016-07-20 13:20:31 +0200310 vnfi = target_dc.startCompute(self.vnf_name2docker_name[vnf_name], network=intfs, image=docker_name, flavor_name="small")
peusterm398cd3b2016-03-21 15:04:54 +0100311 return vnfi
312
peusterm6b5224d2016-07-20 13:20:31 +0200313 def _get_vnf_instance(self, instance_uuid, name):
314 """
315 Returns the Docker object for the given VNF name (or Docker name).
316 :param instance_uuid: UUID of the service instance to search in.
317 :param name: VNF name or Docker name. We are fuzzy here.
318 :return:
319 """
320 dn = name
321 if name in self.vnf_name2docker_name:
322 dn = self.vnf_name2docker_name[name]
323 for vnfi in self.instances[instance_uuid]["vnf_instances"]:
324 if vnfi.name == dn:
325 return vnfi
326 LOG.warning("No container with name: %r found.")
327 return None
328
329 @staticmethod
330 def _vnf_reconfigure_network(vnfi, if_name, net_str):
331 """
332 Reconfigure the network configuration of a specific interface
333 of a running container.
334 :param vnfi: container instacne
335 :param if_name: interface name
336 :param net_str: network configuration string, e.g., 1.2.3.4/24
337 :return:
338 """
339 intf = vnfi.intf(intf=if_name)
340 if intf is not None:
341 intf.setIP(net_str)
342 LOG.debug("Reconfigured network of %s:%s to %r" % (vnfi.name, if_name, net_str))
343 else:
344 LOG.warning("Interface not found: %s:%s. Network reconfiguration skipped." % (vnfi.name, if_name))
345
346
peusterm8484b902016-06-21 09:03:35 +0200347 def _trigger_emulator_start_scripts_in_vnfis(self, vnfi_list):
348 for vnfi in vnfi_list:
349 config = vnfi.dcinfo.get("Config", dict())
350 env = config.get("Env", list())
351 for env_var in env:
352 if "SON_EMU_CMD=" in env_var:
353 cmd = str(env_var.split("=")[1])
peusterme66edf72016-08-23 11:11:12 +0200354 LOG.info("Executing entry point script in %r: %r" % (vnfi.name, cmd))
355 # execute command in new thread to ensure that GK is not blocked by VNF
356 t = threading.Thread(target=vnfi.cmdPrint, args=(cmd,))
357 t.daemon = True
358 t.start()
peusterm8484b902016-06-21 09:03:35 +0200359
peusterm786cd542016-03-14 14:12:17 +0100360 def _unpack_service_package(self):
361 """
362 unzip *.son file and store contents in CATALOG_FOLDER/services/<service_uuid>/
363 """
peusterm82d406e2016-05-02 20:52:06 +0200364 LOG.info("Unzipping: %r" % self.package_file_path)
peusterm786cd542016-03-14 14:12:17 +0100365 with zipfile.ZipFile(self.package_file_path, "r") as z:
366 z.extractall(self.package_content_path)
367
peusterm82d406e2016-05-02 20:52:06 +0200368
peusterm7ec665d2016-03-14 15:20:44 +0100369 def _load_package_descriptor(self):
370 """
371 Load the main package descriptor YAML and keep it as dict.
372 :return:
373 """
374 self.manifest = load_yaml(
375 os.path.join(
376 self.package_content_path, "META-INF/MANIFEST.MF"))
377
378 def _load_nsd(self):
379 """
380 Load the entry NSD YAML and keep it as dict.
381 :return:
382 """
383 if "entry_service_template" in self.manifest:
384 nsd_path = os.path.join(
385 self.package_content_path,
386 make_relative_path(self.manifest.get("entry_service_template")))
387 self.nsd = load_yaml(nsd_path)
peusterm757fe9a2016-04-04 14:11:58 +0200388 LOG.debug("Loaded NSD: %r" % self.nsd.get("name"))
peusterm7ec665d2016-03-14 15:20:44 +0100389
390 def _load_vnfd(self):
391 """
392 Load all VNFD YAML files referenced in MANIFEST.MF and keep them in dict.
393 :return:
394 """
395 if "package_content" in self.manifest:
396 for pc in self.manifest.get("package_content"):
397 if pc.get("content-type") == "application/sonata.function_descriptor":
398 vnfd_path = os.path.join(
399 self.package_content_path,
400 make_relative_path(pc.get("name")))
401 vnfd = load_yaml(vnfd_path)
peusterm757fe9a2016-04-04 14:11:58 +0200402 self.vnfds[vnfd.get("name")] = vnfd
403 LOG.debug("Loaded VNFD: %r" % vnfd.get("name"))
peusterm7ec665d2016-03-14 15:20:44 +0100404
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200405 def _load_saps(self):
406 # Each Service Access Point (connection_point) in the nsd is getting its own container
407 SAPs = [p["id"] for p in self.nsd["connection_points"] if p["type"] == "interface"]
408 for sap in SAPs:
409 # endpoints needed in this service
410 sap_vnf_id, sap_vnf_interface = sap.split(':')
411 # set of the connection_point ids found in the nsd (in the examples this is 'ns')
412 self.sap_identifiers.add(sap_vnf_id)
413
peusterm76eb8652016-09-06 11:07:16 +0200414 sap_docker_name = "%s_%s" % (sap_vnf_id, sap_vnf_interface)
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200415
416 # add SAP to self.vnfds
417 sapfile = pkg_resources.resource_filename(__name__, "sap_vnfd.yml")
418 sap_vnfd = load_yaml(sapfile)
419 sap_vnfd["connection_points"][0]["id"] = sap_vnf_interface
420 sap_vnfd["name"] = sap_docker_name
421 self.vnfds[sap_docker_name] = sap_vnfd
422 # add SAP vnf to list in the NSD so it is deployed later on
423 # each SAP get a unique VNFD and vnf_id in the NSD
424 self.nsd["network_functions"].append({"vnf_id": sap_docker_name, "vnf_name": sap_docker_name})
425 LOG.debug("Loaded SAP: %r" % sap_vnfd.get("name"))
426
peusterm7ec665d2016-03-14 15:20:44 +0100427 def _load_docker_files(self):
428 """
peusterm9d7d4b02016-03-23 19:56:44 +0100429 Get all paths to Dockerfiles from VNFDs and store them in dict.
peusterm7ec665d2016-03-14 15:20:44 +0100430 :return:
431 """
peusterm9d7d4b02016-03-23 19:56:44 +0100432 for k, v in self.vnfds.iteritems():
433 for vu in v.get("virtual_deployment_units"):
434 if vu.get("vm_image_format") == "docker":
435 vm_image = vu.get("vm_image")
peusterm7ec665d2016-03-14 15:20:44 +0100436 docker_path = os.path.join(
437 self.package_content_path,
peusterm9d7d4b02016-03-23 19:56:44 +0100438 make_relative_path(vm_image))
439 self.local_docker_files[k] = docker_path
peusterm56356cb2016-05-03 10:43:43 +0200440 LOG.debug("Found Dockerfile (%r): %r" % (k, docker_path))
peusterm7ec665d2016-03-14 15:20:44 +0100441
peusterm82d406e2016-05-02 20:52:06 +0200442 def _load_docker_urls(self):
443 """
444 Get all URLs to pre-build docker images in some repo.
445 :return:
446 """
447 for k, v in self.vnfds.iteritems():
448 for vu in v.get("virtual_deployment_units"):
449 if vu.get("vm_image_format") == "docker":
peusterm35ba4052016-05-02 21:21:14 +0200450 url = vu.get("vm_image")
451 if url is not None:
452 url = url.replace("http://", "")
453 self.remote_docker_image_urls[k] = url
peusterm56356cb2016-05-03 10:43:43 +0200454 LOG.debug("Found Docker image URL (%r): %r" % (k, self.remote_docker_image_urls[k]))
peusterm82d406e2016-05-02 20:52:06 +0200455
peustermbdfab7e2016-03-14 16:03:30 +0100456 def _build_images_from_dockerfiles(self):
457 """
458 Build Docker images for each local Dockerfile found in the package: self.local_docker_files
459 """
peusterm398cd3b2016-03-21 15:04:54 +0100460 if GK_STANDALONE_MODE:
461 return # do not build anything in standalone mode
peustermbdfab7e2016-03-14 16:03:30 +0100462 dc = DockerClient()
463 LOG.info("Building %d Docker images (this may take several minutes) ..." % len(self.local_docker_files))
464 for k, v in self.local_docker_files.iteritems():
465 for line in dc.build(path=v.replace("Dockerfile", ""), tag=k, rm=False, nocache=False):
466 LOG.debug("DOCKER BUILD: %s" % line)
467 LOG.info("Docker image created: %s" % k)
468
peusterm82d406e2016-05-02 20:52:06 +0200469 def _pull_predefined_dockerimages(self):
peustermbdfab7e2016-03-14 16:03:30 +0100470 """
471 If the package contains URLs to pre-build Docker images, we download them with this method.
472 """
peusterm35ba4052016-05-02 21:21:14 +0200473 dc = DockerClient()
474 for url in self.remote_docker_image_urls.itervalues():
peusterm56356cb2016-05-03 10:43:43 +0200475 if not FORCE_PULL: # only pull if not present (speedup for development)
476 if len(dc.images(name=url)) > 0:
477 LOG.debug("Image %r present. Skipping pull." % url)
478 continue
peusterm35ba4052016-05-02 21:21:14 +0200479 LOG.info("Pulling image: %r" % url)
480 dc.pull(url,
481 insecure_registry=True)
peusterm786cd542016-03-14 14:12:17 +0100482
peusterm3444ae42016-03-16 20:46:41 +0100483 def _check_docker_image_exists(self, image_name):
peusterm3f307142016-03-16 21:02:53 +0100484 """
485 Query the docker service and check if the given image exists
486 :param image_name: name of the docker image
487 :return:
488 """
489 return len(DockerClient().images(image_name)) > 0
peusterm3444ae42016-03-16 20:46:41 +0100490
peusterm082378b2016-03-16 20:14:22 +0100491 def _calculate_placement(self, algorithm):
492 """
493 Do placement by adding the a field "dc" to
494 each VNFD that points to one of our
495 data center objects known to the gatekeeper.
496 """
497 assert(len(self.vnfds) > 0)
498 assert(len(GK.dcs) > 0)
499 # instantiate algorithm an place
500 p = algorithm()
501 p.place(self.nsd, self.vnfds, GK.dcs)
502 LOG.info("Using placement algorithm: %r" % p.__class__.__name__)
503 # lets print the placement result
504 for name, vnfd in self.vnfds.iteritems():
505 LOG.info("Placed VNF %r on DC %r" % (name, str(vnfd.get("dc"))))
506
507
508"""
509Some (simple) placement algorithms
510"""
511
512
513class FirstDcPlacement(object):
514 """
515 Placement: Always use one and the same data center from the GK.dcs dict.
516 """
517 def place(self, nsd, vnfds, dcs):
518 for name, vnfd in vnfds.iteritems():
519 vnfd["dc"] = list(dcs.itervalues())[0]
520
peusterme26487b2016-03-08 14:00:21 +0100521
peustermf6459542016-08-31 19:00:17 +0200522class RoundRobinDcPlacement(object):
523 """
524 Placement: Distribute VNFs across all available DCs in a round robin fashion.
525 """
peustermf6459542016-08-31 19:00:17 +0200526 def place(self, nsd, vnfds, dcs):
527 c = 0
528 dcs_list = list(dcs.itervalues())
529 for name, vnfd in vnfds.iteritems():
530 vnfd["dc"] = dcs_list[c % len(dcs_list)]
531 c += 1 # inc. c to use next DC
532
533
534
535
peusterme26487b2016-03-08 14:00:21 +0100536"""
537Resource definitions and API endpoints
538"""
539
540
541class Packages(fr.Resource):
542
543 def post(self):
544 """
peusterm26455852016-03-08 14:23:53 +0100545 Upload a *.son service package to the dummy gatekeeper.
546
peusterme26487b2016-03-08 14:00:21 +0100547 We expect request with a *.son file and store it in UPLOAD_FOLDER
peusterm26455852016-03-08 14:23:53 +0100548 :return: UUID
peusterme26487b2016-03-08 14:00:21 +0100549 """
550 try:
551 # get file contents
wtavernib8d9ecb2016-03-25 15:18:31 +0100552 print(request.files)
peusterm593ca582016-03-30 19:55:01 +0200553 # lets search for the package in the request
554 if "package" in request.files:
555 son_file = request.files["package"]
556 # elif "file" in request.files:
557 # son_file = request.files["file"]
558 else:
559 return {"service_uuid": None, "size": 0, "sha1": None, "error": "upload failed. file not found."}, 500
peusterme26487b2016-03-08 14:00:21 +0100560 # generate a uuid to reference this package
561 service_uuid = str(uuid.uuid4())
peusterm786cd542016-03-14 14:12:17 +0100562 file_hash = hashlib.sha1(str(son_file)).hexdigest()
peusterme26487b2016-03-08 14:00:21 +0100563 # ensure that upload folder exists
564 ensure_dir(UPLOAD_FOLDER)
565 upload_path = os.path.join(UPLOAD_FOLDER, "%s.son" % service_uuid)
566 # store *.son file to disk
peusterm786cd542016-03-14 14:12:17 +0100567 son_file.save(upload_path)
peusterme26487b2016-03-08 14:00:21 +0100568 size = os.path.getsize(upload_path)
peusterm786cd542016-03-14 14:12:17 +0100569 # create a service object and register it
570 s = Service(service_uuid, file_hash, upload_path)
571 GK.register_service_package(service_uuid, s)
peusterme26487b2016-03-08 14:00:21 +0100572 # generate the JSON result
peusterm938143e2016-09-15 15:39:36 +0200573 return {"service_uuid": service_uuid, "size": size, "sha1": file_hash, "error": None}, 201
peusterme26487b2016-03-08 14:00:21 +0100574 except Exception as ex:
peusterm786cd542016-03-14 14:12:17 +0100575 LOG.exception("Service package upload failed:")
peusterm593ca582016-03-30 19:55:01 +0200576 return {"service_uuid": None, "size": 0, "sha1": None, "error": "upload failed"}, 500
peusterme26487b2016-03-08 14:00:21 +0100577
578 def get(self):
peusterm26455852016-03-08 14:23:53 +0100579 """
580 Return a list of UUID's of uploaded service packages.
581 :return: dict/list
582 """
peusterm075b46a2016-07-20 17:08:00 +0200583 LOG.info("GET /packages")
peusterm786cd542016-03-14 14:12:17 +0100584 return {"service_uuid_list": list(GK.services.iterkeys())}
peusterme26487b2016-03-08 14:00:21 +0100585
586
587class Instantiations(fr.Resource):
588
589 def post(self):
peusterm26455852016-03-08 14:23:53 +0100590 """
591 Instantiate a service specified by its UUID.
592 Will return a new UUID to identify the running service instance.
593 :return: UUID
594 """
peusterm64b45502016-03-16 21:15:14 +0100595 # try to extract the service uuid from the request
peusterm26455852016-03-08 14:23:53 +0100596 json_data = request.get_json(force=True)
peusterm64b45502016-03-16 21:15:14 +0100597 service_uuid = json_data.get("service_uuid")
598
599 # lets be a bit fuzzy here to make testing easier
600 if service_uuid is None and len(GK.services) > 0:
601 # if we don't get a service uuid, we simple start the first service in the list
602 service_uuid = list(GK.services.iterkeys())[0]
603
peustermbea87372016-03-16 19:37:35 +0100604 if service_uuid in GK.services:
peusterm64b45502016-03-16 21:15:14 +0100605 # ok, we have a service uuid, lets start the service
peustermbea87372016-03-16 19:37:35 +0100606 service_instance_uuid = GK.services.get(service_uuid).start_service()
peusterm26455852016-03-08 14:23:53 +0100607 return {"service_instance_uuid": service_instance_uuid}
peustermbea87372016-03-16 19:37:35 +0100608 return "Service not found", 404
peusterme26487b2016-03-08 14:00:21 +0100609
610 def get(self):
peusterm26455852016-03-08 14:23:53 +0100611 """
612 Returns a list of UUIDs containing all running services.
613 :return: dict / list
614 """
peusterm075b46a2016-07-20 17:08:00 +0200615 LOG.info("GET /instantiations")
616 return {"service_instantiations_list": [
peusterm64b45502016-03-16 21:15:14 +0100617 list(s.instances.iterkeys()) for s in GK.services.itervalues()]}
peusterm786cd542016-03-14 14:12:17 +0100618
peusterme26487b2016-03-08 14:00:21 +0100619
620# create a single, global GK object
621GK = Gatekeeper()
622# setup Flask
623app = Flask(__name__)
624app.config['MAX_CONTENT_LENGTH'] = 512 * 1024 * 1024 # 512 MB max upload
625api = fr.Api(app)
626# define endpoints
peusterm593ca582016-03-30 19:55:01 +0200627api.add_resource(Packages, '/packages')
628api.add_resource(Instantiations, '/instantiations')
peusterme26487b2016-03-08 14:00:21 +0100629
630
peusterm082378b2016-03-16 20:14:22 +0100631def start_rest_api(host, port, datacenters=dict()):
peustermbea87372016-03-16 19:37:35 +0100632 GK.dcs = datacenters
peusterme26487b2016-03-08 14:00:21 +0100633 # start the Flask server (not the best performance but ok for our use case)
634 app.run(host=host,
635 port=port,
636 debug=True,
637 use_reloader=False # this is needed to run Flask in a non-main thread
638 )
639
640
641def ensure_dir(name):
642 if not os.path.exists(name):
peusterm7ec665d2016-03-14 15:20:44 +0100643 os.makedirs(name)
644
645
646def load_yaml(path):
647 with open(path, "r") as f:
648 try:
649 r = yaml.load(f)
650 except yaml.YAMLError as exc:
651 LOG.exception("YAML parse error")
652 r = dict()
653 return r
654
655
656def make_relative_path(path):
peusterm9d7d4b02016-03-23 19:56:44 +0100657 if path.startswith("file://"):
658 path = path.replace("file://", "", 1)
peusterm7ec665d2016-03-14 15:20:44 +0100659 if path.startswith("/"):
peusterm9d7d4b02016-03-23 19:56:44 +0100660 path = path.replace("/", "", 1)
peusterm7ec665d2016-03-14 15:20:44 +0100661 return path
662
663
stevenvanrossem6d5019a2016-08-12 23:00:22 +0200664def generate_lan_string(prefix, base, subnet_size=24, ip=0):
665 """
666 Helper to generate different network configuration strings.
667 """
668 r = "%s.%d.%d/%d" % (prefix, base, ip, subnet_size)
669 return r
670
671
peusterm6b5224d2016-07-20 13:20:31 +0200672def generate_subnet_strings(n, start=1, subnet_size=24, ip=0):
673 """
674 Helper to generate different network configuration strings.
675 """
676 r = list()
677 for i in range(start, start + n):
678 r.append("%d.0.0.%d/%d" % (i, ip, subnet_size))
679 return r
680
681
peusterme26487b2016-03-08 14:00:21 +0100682if __name__ == '__main__':
683 """
684 Lets allow to run the API in standalone mode.
685 """
peusterm398cd3b2016-03-21 15:04:54 +0100686 GK_STANDALONE_MODE = True
peusterme26487b2016-03-08 14:00:21 +0100687 logging.getLogger("werkzeug").setLevel(logging.INFO)
688 start_rest_api("0.0.0.0", 8000)
689