blob: 735860c9028f8ce66ad33ea15910d545e2878125 [file] [log] [blame]
peusterme26487b2016-03-08 14:00:21 +01001"""
peusterm79ef6ae2016-07-08 13:53:57 +02002Copyright (c) 2015 SONATA-NFV and Paderborn University
3ALL RIGHTS RESERVED.
4
5Licensed under the Apache License, Version 2.0 (the "License");
6you may not use this file except in compliance with the License.
7You may obtain a copy of the License at
8
9 http://www.apache.org/licenses/LICENSE-2.0
10
11Unless required by applicable law or agreed to in writing, software
12distributed under the License is distributed on an "AS IS" BASIS,
13WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14See the License for the specific language governing permissions and
15limitations under the License.
16
17Neither the name of the SONATA-NFV [, ANY ADDITIONAL AFFILIATION]
18nor the names of its contributors may be used to endorse or promote
19products derived from this software without specific prior written
20permission.
21
22This work has been performed in the framework of the SONATA project,
23funded by the European Commission under Grant number 671517 through
24the Horizon 2020 and 5G-PPP programmes. The authors would like to
25acknowledge the contributions of their colleagues of the SONATA
26partner consortium (www.sonata-nfv.eu).
27"""
28"""
peusterme26487b2016-03-08 14:00:21 +010029This module implements a simple REST API that behaves like SONATA's gatekeeper.
30
31It is only used to support the development of SONATA's SDK tools and to demonstrate
32the year 1 version of the emulator until the integration with WP4's orchestrator is done.
33"""
34
35import logging
36import os
37import uuid
38import hashlib
peusterm786cd542016-03-14 14:12:17 +010039import zipfile
peusterm7ec665d2016-03-14 15:20:44 +010040import yaml
peusterme66edf72016-08-23 11:11:12 +020041import threading
peustermbdfab7e2016-03-14 16:03:30 +010042from docker import Client as DockerClient
peusterme26487b2016-03-08 14:00:21 +010043from flask import Flask, request
44import flask_restful as fr
wtaverni5b23b662016-06-20 12:26:21 +020045from collections import defaultdict
stevenvanrossemdb2f9432016-08-20 00:01:11 +020046import pkg_resources
peusterme26487b2016-03-08 14:00:21 +010047
peusterm398cd3b2016-03-21 15:04:54 +010048logging.basicConfig()
peusterm786cd542016-03-14 14:12:17 +010049LOG = logging.getLogger("sonata-dummy-gatekeeper")
50LOG.setLevel(logging.DEBUG)
peusterme26487b2016-03-08 14:00:21 +010051logging.getLogger("werkzeug").setLevel(logging.WARNING)
52
peusterm92237dc2016-03-21 15:45:58 +010053GK_STORAGE = "/tmp/son-dummy-gk/"
54UPLOAD_FOLDER = os.path.join(GK_STORAGE, "uploads/")
55CATALOG_FOLDER = os.path.join(GK_STORAGE, "catalog/")
peusterme26487b2016-03-08 14:00:21 +010056
peusterm82d406e2016-05-02 20:52:06 +020057# Enable Dockerfile build functionality
58BUILD_DOCKERFILE = False
59
peusterm398cd3b2016-03-21 15:04:54 +010060# flag to indicate that we run without the emulator (only the bare API for integration testing)
61GK_STANDALONE_MODE = False
62
peusterm56356cb2016-05-03 10:43:43 +020063# should a new version of an image be pulled even if its available
wtaverni5b23b662016-06-20 12:26:21 +020064FORCE_PULL = False
peusterme26487b2016-03-08 14:00:21 +010065
stevenvanrossemdb2f9432016-08-20 00:01:11 +020066# Automatically deploy SAPs (endpoints) of the service as new containers
peustermb1cf5372016-08-23 14:02:09 +020067# Attention: This is not a configuration switch but a global variable! Don't change its default value.
stevenvanrossemdb2f9432016-08-20 00:01:11 +020068DEPLOY_SAP = False
69
peusterm76eb8652016-09-06 11:07:16 +020070# flag to indicate if we use bidirectional forwarding rules in the automatic chaining process
71BIDIRECTIONAL_CHAIN = False
72
peusterme26487b2016-03-08 14:00:21 +010073class Gatekeeper(object):
74
75 def __init__(self):
peusterm786cd542016-03-14 14:12:17 +010076 self.services = dict()
peusterm082378b2016-03-16 20:14:22 +010077 self.dcs = dict()
stevenvanrossembecc7c52016-11-07 05:52:01 +010078 self.net = None
peusterm3444ae42016-03-16 20:46:41 +010079 self.vnf_counter = 0 # used to generate short names for VNFs (Mininet limitation)
peusterm786cd542016-03-14 14:12:17 +010080 LOG.info("Create SONATA dummy gatekeeper.")
peusterme26487b2016-03-08 14:00:21 +010081
peusterm786cd542016-03-14 14:12:17 +010082 def register_service_package(self, service_uuid, service):
83 """
84 register new service package
85 :param service_uuid
86 :param service object
87 """
88 self.services[service_uuid] = service
89 # lets perform all steps needed to onboard the service
90 service.onboard()
91
peusterm3444ae42016-03-16 20:46:41 +010092 def get_next_vnf_name(self):
93 self.vnf_counter += 1
peusterm398cd3b2016-03-21 15:04:54 +010094 return "vnf%d" % self.vnf_counter
peusterm3444ae42016-03-16 20:46:41 +010095
peusterm786cd542016-03-14 14:12:17 +010096
97class Service(object):
98 """
99 This class represents a NS uploaded as a *.son package to the
100 dummy gatekeeper.
101 Can have multiple running instances of this service.
102 """
103
104 def __init__(self,
105 service_uuid,
106 package_file_hash,
107 package_file_path):
108 self.uuid = service_uuid
109 self.package_file_hash = package_file_hash
110 self.package_file_path = package_file_path
111 self.package_content_path = os.path.join(CATALOG_FOLDER, "services/%s" % self.uuid)
peusterm7ec665d2016-03-14 15:20:44 +0100112 self.manifest = None
113 self.nsd = None
114 self.vnfds = dict()
peustermbdfab7e2016-03-14 16:03:30 +0100115 self.local_docker_files = dict()
peusterm82d406e2016-05-02 20:52:06 +0200116 self.remote_docker_image_urls = dict()
peusterm786cd542016-03-14 14:12:17 +0100117 self.instances = dict()
peusterm6b5224d2016-07-20 13:20:31 +0200118 self.vnf_name2docker_name = dict()
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200119 self.sap_identifiers = set()
peusterm6b5224d2016-07-20 13:20:31 +0200120 # lets generate a set of subnet configurations used for e-line chaining setup
121 self.eline_subnets_src = generate_subnet_strings(50, start=200, subnet_size=24, ip=1)
122 self.eline_subnets_dst = generate_subnet_strings(50, start=200, subnet_size=24, ip=2)
peusterme26487b2016-03-08 14:00:21 +0100123
peusterm786cd542016-03-14 14:12:17 +0100124 def onboard(self):
125 """
126 Do all steps to prepare this service to be instantiated
127 :return:
128 """
129 # 1. extract the contents of the package and store them in our catalog
130 self._unpack_service_package()
131 # 2. read in all descriptor files
peusterm7ec665d2016-03-14 15:20:44 +0100132 self._load_package_descriptor()
133 self._load_nsd()
134 self._load_vnfd()
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200135 if DEPLOY_SAP:
136 self._load_saps()
peusterm786cd542016-03-14 14:12:17 +0100137 # 3. prepare container images (e.g. download or build Dockerfile)
peusterm82d406e2016-05-02 20:52:06 +0200138 if BUILD_DOCKERFILE:
139 self._load_docker_files()
140 self._build_images_from_dockerfiles()
141 else:
142 self._load_docker_urls()
143 self._pull_predefined_dockerimages()
peusterm3bb86bf2016-08-15 09:47:57 +0200144 LOG.info("On-boarded service: %r" % self.manifest.get("name"))
peusterm7ec665d2016-03-14 15:20:44 +0100145
peusterm082378b2016-03-16 20:14:22 +0100146 def start_service(self):
peusterm3444ae42016-03-16 20:46:41 +0100147 """
148 This methods creates and starts a new service instance.
149 It computes placements, iterates over all VNFDs, and starts
150 each VNFD as a Docker container in the data center selected
151 by the placement algorithm.
152 :return:
153 """
154 LOG.info("Starting service %r" % self.uuid)
stevenvanrossemd87fe472016-05-11 11:34:34 +0200155
peusterm3444ae42016-03-16 20:46:41 +0100156 # 1. each service instance gets a new uuid to identify it
peusterm082378b2016-03-16 20:14:22 +0100157 instance_uuid = str(uuid.uuid4())
peusterm3444ae42016-03-16 20:46:41 +0100158 # build a instances dict (a bit like a NSR :))
159 self.instances[instance_uuid] = dict()
160 self.instances[instance_uuid]["vnf_instances"] = list()
stevenvanrossemd87fe472016-05-11 11:34:34 +0200161
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200162 # 2. Configure the chaining of the network functions (currently only E-Line and E-LAN links supported)
163 vnf_id2vnf_name = defaultdict(lambda: "NotExistingNode",
164 reduce(lambda x, y: dict(x, **y),
165 map(lambda d: {d["vnf_id"]: d["vnf_name"]},
166 self.nsd["network_functions"])))
167
168 # 3. compute placement of this service instance (adds DC names to VNFDs)
peusterm398cd3b2016-03-21 15:04:54 +0100169 if not GK_STANDALONE_MODE:
peustermf6459542016-08-31 19:00:17 +0200170 #self._calculate_placement(FirstDcPlacement)
171 self._calculate_placement(RoundRobinDcPlacement)
peusterm3444ae42016-03-16 20:46:41 +0100172 # iterate over all vnfds that we have to start
peusterm082378b2016-03-16 20:14:22 +0100173 for vnfd in self.vnfds.itervalues():
peusterm398cd3b2016-03-21 15:04:54 +0100174 vnfi = None
175 if not GK_STANDALONE_MODE:
176 vnfi = self._start_vnfd(vnfd)
177 self.instances[instance_uuid]["vnf_instances"].append(vnfi)
stevenvanrossemd87fe472016-05-11 11:34:34 +0200178
stevenvanrossemd87fe472016-05-11 11:34:34 +0200179 vlinks = self.nsd["virtual_links"]
180 fwd_links = self.nsd["forwarding_graphs"][0]["constituent_virtual_links"]
181 eline_fwd_links = [l for l in vlinks if (l["id"] in fwd_links) and (l["connectivity_type"] == "E-Line")]
stevenvanrossem6d5019a2016-08-12 23:00:22 +0200182 elan_fwd_links = [l for l in vlinks if (l["id"] in fwd_links) and (l["connectivity_type"] == "E-LAN")]
stevenvanrossemd87fe472016-05-11 11:34:34 +0200183
stevenvanrossembecc7c52016-11-07 05:52:01 +0100184 GK.net.deployed_elines.extend(eline_fwd_links)
185 GK.net.deployed_elans.extend(elan_fwd_links)
186
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200187 # 4a. deploy E-Line links
stevenvanrossemaa6d3a72016-08-10 13:23:24 +0200188 # cookie is used as identifier for the flowrules installed by the dummygatekeeper
189 # eg. different services get a unique cookie for their flowrules
190 cookie = 1
stevenvanrossemd87fe472016-05-11 11:34:34 +0200191 for link in eline_fwd_links:
peusterm6b5224d2016-07-20 13:20:31 +0200192 src_id, src_if_name = link["connection_points_reference"][0].split(":")
193 dst_id, dst_if_name = link["connection_points_reference"][1].split(":")
stevenvanrossemd87fe472016-05-11 11:34:34 +0200194
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200195 # check if there is a SAP in the link
196 if src_id in self.sap_identifiers:
197 src_docker_name = "{0}_{1}".format(src_id, src_if_name)
198 src_id = src_docker_name
199 else:
200 src_docker_name = src_id
201
202 if dst_id in self.sap_identifiers:
203 dst_docker_name = "{0}_{1}".format(dst_id, dst_if_name)
204 dst_id = dst_docker_name
205 else:
206 dst_docker_name = dst_id
207
peusterm6b5224d2016-07-20 13:20:31 +0200208 src_name = vnf_id2vnf_name[src_id]
209 dst_name = vnf_id2vnf_name[dst_id]
peusterm9fb74ec2016-06-16 11:30:55 +0200210
peusterm6b5224d2016-07-20 13:20:31 +0200211 LOG.debug(
212 "Setting up E-Line link. %s(%s:%s) -> %s(%s:%s)" % (
213 src_name, src_id, src_if_name, dst_name, dst_id, dst_if_name))
214
215 if (src_name in self.vnfds) and (dst_name in self.vnfds):
216 network = self.vnfds[src_name].get("dc").net # there should be a cleaner way to find the DCNetwork
peusterm6b5224d2016-07-20 13:20:31 +0200217 LOG.debug(src_docker_name)
218 ret = network.setChain(
219 src_docker_name, dst_docker_name,
220 vnf_src_interface=src_if_name, vnf_dst_interface=dst_if_name,
peusterm76eb8652016-09-06 11:07:16 +0200221 bidirectional=BIDIRECTIONAL_CHAIN, cmd="add-flow", cookie=cookie, priority=10)
stevenvanrossemd87fe472016-05-11 11:34:34 +0200222
peusterm6b5224d2016-07-20 13:20:31 +0200223 # re-configure the VNFs IP assignment and ensure that a new subnet is used for each E-Link
224 src_vnfi = self._get_vnf_instance(instance_uuid, src_name)
225 if src_vnfi is not None:
226 self._vnf_reconfigure_network(src_vnfi, src_if_name, self.eline_subnets_src.pop(0))
227 dst_vnfi = self._get_vnf_instance(instance_uuid, dst_name)
228 if dst_vnfi is not None:
229 self._vnf_reconfigure_network(dst_vnfi, dst_if_name, self.eline_subnets_dst.pop(0))
230
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200231 # 4b. deploy E-LAN links
stevenvanrossem6d5019a2016-08-12 23:00:22 +0200232 base = 10
233 for link in elan_fwd_links:
stevenvanrossemdc3bfd02016-11-04 15:33:28 +0100234
stevenvanrossembecc7c52016-11-07 05:52:01 +0100235 elan_vnf_list=[]
stevenvanrossemdc3bfd02016-11-04 15:33:28 +0100236
stevenvanrossem6d5019a2016-08-12 23:00:22 +0200237 # generate lan ip address
238 ip = 1
239 for intf in link["connection_points_reference"]:
240 ip_address = generate_lan_string("10.0", base, subnet_size=24, ip=ip)
241 vnf_id, intf_name = intf.split(":")
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200242 if vnf_id in self.sap_identifiers:
243 src_docker_name = "{0}_{1}".format(vnf_id, intf_name)
244 vnf_id = src_docker_name
stevenvanrossemdc3bfd02016-11-04 15:33:28 +0100245 else:
246 src_docker_name = vnf_id
stevenvanrossem6d5019a2016-08-12 23:00:22 +0200247 vnf_name = vnf_id2vnf_name[vnf_id]
248 LOG.debug(
249 "Setting up E-LAN link. %s(%s:%s) -> %s" % (
250 vnf_name, vnf_id, intf_name, ip_address))
251
252 if vnf_name in self.vnfds:
253 # re-configure the VNFs IP assignment and ensure that a new subnet is used for each E-LAN
peustermb1cf5372016-08-23 14:02:09 +0200254 # E-LAN relies on the learning switch capability of Ryu which has to be turned on in the topology
255 # (DCNetwork(controller=RemoteController, enable_learning=True)), so no explicit chaining is necessary.
stevenvanrossem6d5019a2016-08-12 23:00:22 +0200256 vnfi = self._get_vnf_instance(instance_uuid, vnf_name)
257 if vnfi is not None:
258 self._vnf_reconfigure_network(vnfi, intf_name, ip_address)
259 # increase for the next ip address on this E-LAN
260 ip += 1
stevenvanrossemdc3bfd02016-11-04 15:33:28 +0100261
262 # add this vnf and interface to the E-LAN for tagging
263 network = self.vnfds[vnf_name].get("dc").net # there should be a cleaner way to find the DCNetwork
stevenvanrossembecc7c52016-11-07 05:52:01 +0100264 elan_vnf_list.append({'name':src_docker_name,'interface':intf_name})
stevenvanrossemdc3bfd02016-11-04 15:33:28 +0100265
266
267 # install the VLAN tags for this E-LAN
stevenvanrossembecc7c52016-11-07 05:52:01 +0100268 network.setLAN(elan_vnf_list)
stevenvanrossem6d5019a2016-08-12 23:00:22 +0200269 # increase the base ip address for the next E-LAN
270 base += 1
271
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200272 # 5. run the emulator specific entrypoint scripts in the VNFIs of this service instance
peusterm8484b902016-06-21 09:03:35 +0200273 self._trigger_emulator_start_scripts_in_vnfis(self.instances[instance_uuid]["vnf_instances"])
274
peusterm3444ae42016-03-16 20:46:41 +0100275 LOG.info("Service started. Instance id: %r" % instance_uuid)
peusterm082378b2016-03-16 20:14:22 +0100276 return instance_uuid
277
peusterm398cd3b2016-03-21 15:04:54 +0100278 def _start_vnfd(self, vnfd):
279 """
280 Start a single VNFD of this service
281 :param vnfd: vnfd descriptor dict
282 :return:
283 """
284 # iterate over all deployment units within each VNFDs
285 for u in vnfd.get("virtual_deployment_units"):
286 # 1. get the name of the docker image to start and the assigned DC
peusterm56356cb2016-05-03 10:43:43 +0200287 vnf_name = vnfd.get("name")
288 if vnf_name not in self.remote_docker_image_urls:
289 raise Exception("No image name for %r found. Abort." % vnf_name)
290 docker_name = self.remote_docker_image_urls.get(vnf_name)
peusterm398cd3b2016-03-21 15:04:54 +0100291 target_dc = vnfd.get("dc")
292 # 2. perform some checks to ensure we can start the container
293 assert(docker_name is not None)
294 assert(target_dc is not None)
295 if not self._check_docker_image_exists(docker_name):
296 raise Exception("Docker image %r not found. Abort." % docker_name)
297 # 3. do the dc.startCompute(name="foobar") call to run the container
298 # TODO consider flavors, and other annotations
stevenvanrossemd87fe472016-05-11 11:34:34 +0200299 intfs = vnfd.get("connection_points")
stevenvanrossemeae73082016-08-05 16:22:12 +0200300
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200301 # TODO: get all vnf id's from the nsd for this vnfd and use those as dockername
stevenvanrossem11a021f2016-08-05 13:43:00 +0200302 # use the vnf_id in the nsd as docker name
303 # so deployed containers can be easily mapped back to the nsd
304 vnf_name2id = defaultdict(lambda: "NotExistingNode",
305 reduce(lambda x, y: dict(x, **y),
306 map(lambda d: {d["vnf_name"]: d["vnf_id"]},
307 self.nsd["network_functions"])))
308 self.vnf_name2docker_name[vnf_name] = vnf_name2id[vnf_name]
309 # self.vnf_name2docker_name[vnf_name] = GK.get_next_vnf_name()
310
peusterm6b5224d2016-07-20 13:20:31 +0200311 LOG.info("Starting %r as %r in DC %r" % (vnf_name, self.vnf_name2docker_name[vnf_name], vnfd.get("dc")))
peusterm761c14d2016-07-19 09:31:19 +0200312 LOG.debug("Interfaces for %r: %r" % (vnf_name, intfs))
peusterm6b5224d2016-07-20 13:20:31 +0200313 vnfi = target_dc.startCompute(self.vnf_name2docker_name[vnf_name], network=intfs, image=docker_name, flavor_name="small")
peusterm398cd3b2016-03-21 15:04:54 +0100314 return vnfi
315
peusterm6b5224d2016-07-20 13:20:31 +0200316 def _get_vnf_instance(self, instance_uuid, name):
317 """
318 Returns the Docker object for the given VNF name (or Docker name).
319 :param instance_uuid: UUID of the service instance to search in.
320 :param name: VNF name or Docker name. We are fuzzy here.
321 :return:
322 """
323 dn = name
324 if name in self.vnf_name2docker_name:
325 dn = self.vnf_name2docker_name[name]
326 for vnfi in self.instances[instance_uuid]["vnf_instances"]:
327 if vnfi.name == dn:
328 return vnfi
329 LOG.warning("No container with name: %r found.")
330 return None
331
332 @staticmethod
333 def _vnf_reconfigure_network(vnfi, if_name, net_str):
334 """
335 Reconfigure the network configuration of a specific interface
336 of a running container.
337 :param vnfi: container instacne
338 :param if_name: interface name
339 :param net_str: network configuration string, e.g., 1.2.3.4/24
340 :return:
341 """
342 intf = vnfi.intf(intf=if_name)
343 if intf is not None:
344 intf.setIP(net_str)
345 LOG.debug("Reconfigured network of %s:%s to %r" % (vnfi.name, if_name, net_str))
346 else:
347 LOG.warning("Interface not found: %s:%s. Network reconfiguration skipped." % (vnfi.name, if_name))
348
349
peusterm8484b902016-06-21 09:03:35 +0200350 def _trigger_emulator_start_scripts_in_vnfis(self, vnfi_list):
351 for vnfi in vnfi_list:
352 config = vnfi.dcinfo.get("Config", dict())
353 env = config.get("Env", list())
354 for env_var in env:
355 if "SON_EMU_CMD=" in env_var:
356 cmd = str(env_var.split("=")[1])
peusterme66edf72016-08-23 11:11:12 +0200357 LOG.info("Executing entry point script in %r: %r" % (vnfi.name, cmd))
358 # execute command in new thread to ensure that GK is not blocked by VNF
359 t = threading.Thread(target=vnfi.cmdPrint, args=(cmd,))
360 t.daemon = True
361 t.start()
peusterm8484b902016-06-21 09:03:35 +0200362
peusterm786cd542016-03-14 14:12:17 +0100363 def _unpack_service_package(self):
364 """
365 unzip *.son file and store contents in CATALOG_FOLDER/services/<service_uuid>/
366 """
peusterm82d406e2016-05-02 20:52:06 +0200367 LOG.info("Unzipping: %r" % self.package_file_path)
peusterm786cd542016-03-14 14:12:17 +0100368 with zipfile.ZipFile(self.package_file_path, "r") as z:
369 z.extractall(self.package_content_path)
370
peusterm82d406e2016-05-02 20:52:06 +0200371
peusterm7ec665d2016-03-14 15:20:44 +0100372 def _load_package_descriptor(self):
373 """
374 Load the main package descriptor YAML and keep it as dict.
375 :return:
376 """
377 self.manifest = load_yaml(
378 os.path.join(
379 self.package_content_path, "META-INF/MANIFEST.MF"))
380
381 def _load_nsd(self):
382 """
383 Load the entry NSD YAML and keep it as dict.
384 :return:
385 """
386 if "entry_service_template" in self.manifest:
387 nsd_path = os.path.join(
388 self.package_content_path,
389 make_relative_path(self.manifest.get("entry_service_template")))
390 self.nsd = load_yaml(nsd_path)
stevenvanrossembecc7c52016-11-07 05:52:01 +0100391 GK.net.deployed_nsds.append(self.nsd)
peusterm757fe9a2016-04-04 14:11:58 +0200392 LOG.debug("Loaded NSD: %r" % self.nsd.get("name"))
peusterm7ec665d2016-03-14 15:20:44 +0100393
394 def _load_vnfd(self):
395 """
396 Load all VNFD YAML files referenced in MANIFEST.MF and keep them in dict.
397 :return:
398 """
399 if "package_content" in self.manifest:
400 for pc in self.manifest.get("package_content"):
401 if pc.get("content-type") == "application/sonata.function_descriptor":
402 vnfd_path = os.path.join(
403 self.package_content_path,
404 make_relative_path(pc.get("name")))
405 vnfd = load_yaml(vnfd_path)
peusterm757fe9a2016-04-04 14:11:58 +0200406 self.vnfds[vnfd.get("name")] = vnfd
407 LOG.debug("Loaded VNFD: %r" % vnfd.get("name"))
peusterm7ec665d2016-03-14 15:20:44 +0100408
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200409 def _load_saps(self):
410 # Each Service Access Point (connection_point) in the nsd is getting its own container
411 SAPs = [p["id"] for p in self.nsd["connection_points"] if p["type"] == "interface"]
412 for sap in SAPs:
413 # endpoints needed in this service
414 sap_vnf_id, sap_vnf_interface = sap.split(':')
415 # set of the connection_point ids found in the nsd (in the examples this is 'ns')
416 self.sap_identifiers.add(sap_vnf_id)
417
peusterm76eb8652016-09-06 11:07:16 +0200418 sap_docker_name = "%s_%s" % (sap_vnf_id, sap_vnf_interface)
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200419
420 # add SAP to self.vnfds
421 sapfile = pkg_resources.resource_filename(__name__, "sap_vnfd.yml")
422 sap_vnfd = load_yaml(sapfile)
423 sap_vnfd["connection_points"][0]["id"] = sap_vnf_interface
424 sap_vnfd["name"] = sap_docker_name
425 self.vnfds[sap_docker_name] = sap_vnfd
426 # add SAP vnf to list in the NSD so it is deployed later on
427 # each SAP get a unique VNFD and vnf_id in the NSD
428 self.nsd["network_functions"].append({"vnf_id": sap_docker_name, "vnf_name": sap_docker_name})
429 LOG.debug("Loaded SAP: %r" % sap_vnfd.get("name"))
430
peusterm7ec665d2016-03-14 15:20:44 +0100431 def _load_docker_files(self):
432 """
peusterm9d7d4b02016-03-23 19:56:44 +0100433 Get all paths to Dockerfiles from VNFDs and store them in dict.
peusterm7ec665d2016-03-14 15:20:44 +0100434 :return:
435 """
peusterm9d7d4b02016-03-23 19:56:44 +0100436 for k, v in self.vnfds.iteritems():
437 for vu in v.get("virtual_deployment_units"):
438 if vu.get("vm_image_format") == "docker":
439 vm_image = vu.get("vm_image")
peusterm7ec665d2016-03-14 15:20:44 +0100440 docker_path = os.path.join(
441 self.package_content_path,
peusterm9d7d4b02016-03-23 19:56:44 +0100442 make_relative_path(vm_image))
443 self.local_docker_files[k] = docker_path
peusterm56356cb2016-05-03 10:43:43 +0200444 LOG.debug("Found Dockerfile (%r): %r" % (k, docker_path))
peusterm7ec665d2016-03-14 15:20:44 +0100445
peusterm82d406e2016-05-02 20:52:06 +0200446 def _load_docker_urls(self):
447 """
448 Get all URLs to pre-build docker images in some repo.
449 :return:
450 """
451 for k, v in self.vnfds.iteritems():
452 for vu in v.get("virtual_deployment_units"):
453 if vu.get("vm_image_format") == "docker":
peusterm35ba4052016-05-02 21:21:14 +0200454 url = vu.get("vm_image")
455 if url is not None:
456 url = url.replace("http://", "")
457 self.remote_docker_image_urls[k] = url
peusterm56356cb2016-05-03 10:43:43 +0200458 LOG.debug("Found Docker image URL (%r): %r" % (k, self.remote_docker_image_urls[k]))
peusterm82d406e2016-05-02 20:52:06 +0200459
peustermbdfab7e2016-03-14 16:03:30 +0100460 def _build_images_from_dockerfiles(self):
461 """
462 Build Docker images for each local Dockerfile found in the package: self.local_docker_files
463 """
peusterm398cd3b2016-03-21 15:04:54 +0100464 if GK_STANDALONE_MODE:
465 return # do not build anything in standalone mode
peustermbdfab7e2016-03-14 16:03:30 +0100466 dc = DockerClient()
467 LOG.info("Building %d Docker images (this may take several minutes) ..." % len(self.local_docker_files))
468 for k, v in self.local_docker_files.iteritems():
469 for line in dc.build(path=v.replace("Dockerfile", ""), tag=k, rm=False, nocache=False):
470 LOG.debug("DOCKER BUILD: %s" % line)
471 LOG.info("Docker image created: %s" % k)
472
peusterm82d406e2016-05-02 20:52:06 +0200473 def _pull_predefined_dockerimages(self):
peustermbdfab7e2016-03-14 16:03:30 +0100474 """
475 If the package contains URLs to pre-build Docker images, we download them with this method.
476 """
peusterm35ba4052016-05-02 21:21:14 +0200477 dc = DockerClient()
478 for url in self.remote_docker_image_urls.itervalues():
peusterm56356cb2016-05-03 10:43:43 +0200479 if not FORCE_PULL: # only pull if not present (speedup for development)
480 if len(dc.images(name=url)) > 0:
481 LOG.debug("Image %r present. Skipping pull." % url)
482 continue
peusterm35ba4052016-05-02 21:21:14 +0200483 LOG.info("Pulling image: %r" % url)
484 dc.pull(url,
485 insecure_registry=True)
peusterm786cd542016-03-14 14:12:17 +0100486
peusterm3444ae42016-03-16 20:46:41 +0100487 def _check_docker_image_exists(self, image_name):
peusterm3f307142016-03-16 21:02:53 +0100488 """
489 Query the docker service and check if the given image exists
490 :param image_name: name of the docker image
491 :return:
492 """
493 return len(DockerClient().images(image_name)) > 0
peusterm3444ae42016-03-16 20:46:41 +0100494
peusterm082378b2016-03-16 20:14:22 +0100495 def _calculate_placement(self, algorithm):
496 """
497 Do placement by adding the a field "dc" to
498 each VNFD that points to one of our
499 data center objects known to the gatekeeper.
500 """
501 assert(len(self.vnfds) > 0)
502 assert(len(GK.dcs) > 0)
503 # instantiate algorithm an place
504 p = algorithm()
505 p.place(self.nsd, self.vnfds, GK.dcs)
506 LOG.info("Using placement algorithm: %r" % p.__class__.__name__)
507 # lets print the placement result
508 for name, vnfd in self.vnfds.iteritems():
509 LOG.info("Placed VNF %r on DC %r" % (name, str(vnfd.get("dc"))))
510
511
512"""
513Some (simple) placement algorithms
514"""
515
516
517class FirstDcPlacement(object):
518 """
519 Placement: Always use one and the same data center from the GK.dcs dict.
520 """
521 def place(self, nsd, vnfds, dcs):
522 for name, vnfd in vnfds.iteritems():
523 vnfd["dc"] = list(dcs.itervalues())[0]
524
peusterme26487b2016-03-08 14:00:21 +0100525
peustermf6459542016-08-31 19:00:17 +0200526class RoundRobinDcPlacement(object):
527 """
528 Placement: Distribute VNFs across all available DCs in a round robin fashion.
529 """
peustermf6459542016-08-31 19:00:17 +0200530 def place(self, nsd, vnfds, dcs):
531 c = 0
532 dcs_list = list(dcs.itervalues())
533 for name, vnfd in vnfds.iteritems():
534 vnfd["dc"] = dcs_list[c % len(dcs_list)]
535 c += 1 # inc. c to use next DC
536
537
538
539
peusterme26487b2016-03-08 14:00:21 +0100540"""
541Resource definitions and API endpoints
542"""
543
544
545class Packages(fr.Resource):
546
547 def post(self):
548 """
peusterm26455852016-03-08 14:23:53 +0100549 Upload a *.son service package to the dummy gatekeeper.
550
peusterme26487b2016-03-08 14:00:21 +0100551 We expect request with a *.son file and store it in UPLOAD_FOLDER
peusterm26455852016-03-08 14:23:53 +0100552 :return: UUID
peusterme26487b2016-03-08 14:00:21 +0100553 """
554 try:
555 # get file contents
wtavernib8d9ecb2016-03-25 15:18:31 +0100556 print(request.files)
peusterm593ca582016-03-30 19:55:01 +0200557 # lets search for the package in the request
558 if "package" in request.files:
559 son_file = request.files["package"]
560 # elif "file" in request.files:
561 # son_file = request.files["file"]
562 else:
563 return {"service_uuid": None, "size": 0, "sha1": None, "error": "upload failed. file not found."}, 500
peusterme26487b2016-03-08 14:00:21 +0100564 # generate a uuid to reference this package
565 service_uuid = str(uuid.uuid4())
peusterm786cd542016-03-14 14:12:17 +0100566 file_hash = hashlib.sha1(str(son_file)).hexdigest()
peusterme26487b2016-03-08 14:00:21 +0100567 # ensure that upload folder exists
568 ensure_dir(UPLOAD_FOLDER)
569 upload_path = os.path.join(UPLOAD_FOLDER, "%s.son" % service_uuid)
570 # store *.son file to disk
peusterm786cd542016-03-14 14:12:17 +0100571 son_file.save(upload_path)
peusterme26487b2016-03-08 14:00:21 +0100572 size = os.path.getsize(upload_path)
peusterm786cd542016-03-14 14:12:17 +0100573 # create a service object and register it
574 s = Service(service_uuid, file_hash, upload_path)
575 GK.register_service_package(service_uuid, s)
peusterme26487b2016-03-08 14:00:21 +0100576 # generate the JSON result
peusterm938143e2016-09-15 15:39:36 +0200577 return {"service_uuid": service_uuid, "size": size, "sha1": file_hash, "error": None}, 201
peusterme26487b2016-03-08 14:00:21 +0100578 except Exception as ex:
peusterm786cd542016-03-14 14:12:17 +0100579 LOG.exception("Service package upload failed:")
peusterm593ca582016-03-30 19:55:01 +0200580 return {"service_uuid": None, "size": 0, "sha1": None, "error": "upload failed"}, 500
peusterme26487b2016-03-08 14:00:21 +0100581
582 def get(self):
peusterm26455852016-03-08 14:23:53 +0100583 """
584 Return a list of UUID's of uploaded service packages.
585 :return: dict/list
586 """
peusterm075b46a2016-07-20 17:08:00 +0200587 LOG.info("GET /packages")
peusterm786cd542016-03-14 14:12:17 +0100588 return {"service_uuid_list": list(GK.services.iterkeys())}
peusterme26487b2016-03-08 14:00:21 +0100589
590
591class Instantiations(fr.Resource):
592
593 def post(self):
peusterm26455852016-03-08 14:23:53 +0100594 """
595 Instantiate a service specified by its UUID.
596 Will return a new UUID to identify the running service instance.
597 :return: UUID
598 """
peusterm64b45502016-03-16 21:15:14 +0100599 # try to extract the service uuid from the request
peusterm26455852016-03-08 14:23:53 +0100600 json_data = request.get_json(force=True)
peusterm64b45502016-03-16 21:15:14 +0100601 service_uuid = json_data.get("service_uuid")
602
603 # lets be a bit fuzzy here to make testing easier
604 if service_uuid is None and len(GK.services) > 0:
605 # if we don't get a service uuid, we simple start the first service in the list
606 service_uuid = list(GK.services.iterkeys())[0]
607
peustermbea87372016-03-16 19:37:35 +0100608 if service_uuid in GK.services:
peusterm64b45502016-03-16 21:15:14 +0100609 # ok, we have a service uuid, lets start the service
peustermbea87372016-03-16 19:37:35 +0100610 service_instance_uuid = GK.services.get(service_uuid).start_service()
peusterm26455852016-03-08 14:23:53 +0100611 return {"service_instance_uuid": service_instance_uuid}
peustermbea87372016-03-16 19:37:35 +0100612 return "Service not found", 404
peusterme26487b2016-03-08 14:00:21 +0100613
614 def get(self):
peusterm26455852016-03-08 14:23:53 +0100615 """
616 Returns a list of UUIDs containing all running services.
617 :return: dict / list
618 """
peusterm075b46a2016-07-20 17:08:00 +0200619 LOG.info("GET /instantiations")
620 return {"service_instantiations_list": [
peusterm64b45502016-03-16 21:15:14 +0100621 list(s.instances.iterkeys()) for s in GK.services.itervalues()]}
peusterm786cd542016-03-14 14:12:17 +0100622
peusterme26487b2016-03-08 14:00:21 +0100623
624# create a single, global GK object
625GK = Gatekeeper()
626# setup Flask
627app = Flask(__name__)
628app.config['MAX_CONTENT_LENGTH'] = 512 * 1024 * 1024 # 512 MB max upload
629api = fr.Api(app)
630# define endpoints
peusterm593ca582016-03-30 19:55:01 +0200631api.add_resource(Packages, '/packages')
632api.add_resource(Instantiations, '/instantiations')
peusterme26487b2016-03-08 14:00:21 +0100633
634
peusterm082378b2016-03-16 20:14:22 +0100635def start_rest_api(host, port, datacenters=dict()):
peustermbea87372016-03-16 19:37:35 +0100636 GK.dcs = datacenters
stevenvanrossembecc7c52016-11-07 05:52:01 +0100637 GK.net = get_dc_network()
peusterme26487b2016-03-08 14:00:21 +0100638 # start the Flask server (not the best performance but ok for our use case)
639 app.run(host=host,
640 port=port,
641 debug=True,
642 use_reloader=False # this is needed to run Flask in a non-main thread
643 )
644
645
646def ensure_dir(name):
647 if not os.path.exists(name):
peusterm7ec665d2016-03-14 15:20:44 +0100648 os.makedirs(name)
649
650
651def load_yaml(path):
652 with open(path, "r") as f:
653 try:
654 r = yaml.load(f)
655 except yaml.YAMLError as exc:
656 LOG.exception("YAML parse error")
657 r = dict()
658 return r
659
660
661def make_relative_path(path):
peusterm9d7d4b02016-03-23 19:56:44 +0100662 if path.startswith("file://"):
663 path = path.replace("file://", "", 1)
peusterm7ec665d2016-03-14 15:20:44 +0100664 if path.startswith("/"):
peusterm9d7d4b02016-03-23 19:56:44 +0100665 path = path.replace("/", "", 1)
peusterm7ec665d2016-03-14 15:20:44 +0100666 return path
667
668
stevenvanrossem6d5019a2016-08-12 23:00:22 +0200669def generate_lan_string(prefix, base, subnet_size=24, ip=0):
670 """
671 Helper to generate different network configuration strings.
672 """
673 r = "%s.%d.%d/%d" % (prefix, base, ip, subnet_size)
674 return r
675
676
peusterm6b5224d2016-07-20 13:20:31 +0200677def generate_subnet_strings(n, start=1, subnet_size=24, ip=0):
678 """
679 Helper to generate different network configuration strings.
680 """
681 r = list()
682 for i in range(start, start + n):
683 r.append("%d.0.0.%d/%d" % (i, ip, subnet_size))
684 return r
685
stevenvanrossembecc7c52016-11-07 05:52:01 +0100686def get_dc_network():
687 """
688 retrieve the DCnetwork where this dummygatekeeper (GK) connects to.
689 Assume at least 1 datacenter is connected to this GK, and that all datacenters belong to the same DCNetwork
690 :return:
691 """
692 assert (len(GK.dcs) > 0)
693 return GK.dcs.values()[0].net
peusterm6b5224d2016-07-20 13:20:31 +0200694
peusterme26487b2016-03-08 14:00:21 +0100695if __name__ == '__main__':
696 """
697 Lets allow to run the API in standalone mode.
698 """
peusterm398cd3b2016-03-21 15:04:54 +0100699 GK_STANDALONE_MODE = True
peusterme26487b2016-03-08 14:00:21 +0100700 logging.getLogger("werkzeug").setLevel(logging.INFO)
701 start_rest_api("0.0.0.0", 8000)
702