blob: 70fce59696add08f3e55a4d32b5b3cc0d0febf66 [file] [log] [blame]
peusterme26487b2016-03-08 14:00:21 +01001"""
peusterm79ef6ae2016-07-08 13:53:57 +02002Copyright (c) 2015 SONATA-NFV and Paderborn University
3ALL RIGHTS RESERVED.
4
5Licensed under the Apache License, Version 2.0 (the "License");
6you may not use this file except in compliance with the License.
7You may obtain a copy of the License at
8
9 http://www.apache.org/licenses/LICENSE-2.0
10
11Unless required by applicable law or agreed to in writing, software
12distributed under the License is distributed on an "AS IS" BASIS,
13WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14See the License for the specific language governing permissions and
15limitations under the License.
16
17Neither the name of the SONATA-NFV [, ANY ADDITIONAL AFFILIATION]
18nor the names of its contributors may be used to endorse or promote
19products derived from this software without specific prior written
20permission.
21
22This work has been performed in the framework of the SONATA project,
23funded by the European Commission under Grant number 671517 through
24the Horizon 2020 and 5G-PPP programmes. The authors would like to
25acknowledge the contributions of their colleagues of the SONATA
26partner consortium (www.sonata-nfv.eu).
27"""
28"""
peusterme26487b2016-03-08 14:00:21 +010029This module implements a simple REST API that behaves like SONATA's gatekeeper.
30
31It is only used to support the development of SONATA's SDK tools and to demonstrate
32the year 1 version of the emulator until the integration with WP4's orchestrator is done.
33"""
34
35import logging
36import os
37import uuid
38import hashlib
peusterm786cd542016-03-14 14:12:17 +010039import zipfile
peusterm7ec665d2016-03-14 15:20:44 +010040import yaml
peusterme66edf72016-08-23 11:11:12 +020041import threading
peustermbdfab7e2016-03-14 16:03:30 +010042from docker import Client as DockerClient
peusterme26487b2016-03-08 14:00:21 +010043from flask import Flask, request
44import flask_restful as fr
wtaverni5b23b662016-06-20 12:26:21 +020045from collections import defaultdict
stevenvanrossemdb2f9432016-08-20 00:01:11 +020046import pkg_resources
peusterme26487b2016-03-08 14:00:21 +010047
peusterm398cd3b2016-03-21 15:04:54 +010048logging.basicConfig()
peusterm786cd542016-03-14 14:12:17 +010049LOG = logging.getLogger("sonata-dummy-gatekeeper")
50LOG.setLevel(logging.DEBUG)
peusterme26487b2016-03-08 14:00:21 +010051logging.getLogger("werkzeug").setLevel(logging.WARNING)
52
peusterm92237dc2016-03-21 15:45:58 +010053GK_STORAGE = "/tmp/son-dummy-gk/"
54UPLOAD_FOLDER = os.path.join(GK_STORAGE, "uploads/")
55CATALOG_FOLDER = os.path.join(GK_STORAGE, "catalog/")
peusterme26487b2016-03-08 14:00:21 +010056
peusterm82d406e2016-05-02 20:52:06 +020057# Enable Dockerfile build functionality
58BUILD_DOCKERFILE = False
59
peusterm398cd3b2016-03-21 15:04:54 +010060# flag to indicate that we run without the emulator (only the bare API for integration testing)
61GK_STANDALONE_MODE = False
62
peusterm56356cb2016-05-03 10:43:43 +020063# should a new version of an image be pulled even if its available
wtaverni5b23b662016-06-20 12:26:21 +020064FORCE_PULL = False
peusterme26487b2016-03-08 14:00:21 +010065
stevenvanrossemdb2f9432016-08-20 00:01:11 +020066# Automatically deploy SAPs (endpoints) of the service as new containers
peustermb1cf5372016-08-23 14:02:09 +020067# Attention: This is not a configuration switch but a global variable! Don't change its default value.
stevenvanrossemdb2f9432016-08-20 00:01:11 +020068DEPLOY_SAP = False
69
peusterm76eb8652016-09-06 11:07:16 +020070# flag to indicate if we use bidirectional forwarding rules in the automatic chaining process
71BIDIRECTIONAL_CHAIN = False
72
peusterme26487b2016-03-08 14:00:21 +010073class Gatekeeper(object):
74
75 def __init__(self):
peusterm786cd542016-03-14 14:12:17 +010076 self.services = dict()
peusterm082378b2016-03-16 20:14:22 +010077 self.dcs = dict()
peusterm3444ae42016-03-16 20:46:41 +010078 self.vnf_counter = 0 # used to generate short names for VNFs (Mininet limitation)
peusterm786cd542016-03-14 14:12:17 +010079 LOG.info("Create SONATA dummy gatekeeper.")
peusterme26487b2016-03-08 14:00:21 +010080
peusterm786cd542016-03-14 14:12:17 +010081 def register_service_package(self, service_uuid, service):
82 """
83 register new service package
84 :param service_uuid
85 :param service object
86 """
87 self.services[service_uuid] = service
88 # lets perform all steps needed to onboard the service
89 service.onboard()
90
peusterm3444ae42016-03-16 20:46:41 +010091 def get_next_vnf_name(self):
92 self.vnf_counter += 1
peusterm398cd3b2016-03-21 15:04:54 +010093 return "vnf%d" % self.vnf_counter
peusterm3444ae42016-03-16 20:46:41 +010094
peusterm786cd542016-03-14 14:12:17 +010095
96class Service(object):
97 """
98 This class represents a NS uploaded as a *.son package to the
99 dummy gatekeeper.
100 Can have multiple running instances of this service.
101 """
102
103 def __init__(self,
104 service_uuid,
105 package_file_hash,
106 package_file_path):
107 self.uuid = service_uuid
108 self.package_file_hash = package_file_hash
109 self.package_file_path = package_file_path
110 self.package_content_path = os.path.join(CATALOG_FOLDER, "services/%s" % self.uuid)
peusterm7ec665d2016-03-14 15:20:44 +0100111 self.manifest = None
112 self.nsd = None
113 self.vnfds = dict()
peustermbdfab7e2016-03-14 16:03:30 +0100114 self.local_docker_files = dict()
peusterm82d406e2016-05-02 20:52:06 +0200115 self.remote_docker_image_urls = dict()
peusterm786cd542016-03-14 14:12:17 +0100116 self.instances = dict()
peusterm6b5224d2016-07-20 13:20:31 +0200117 self.vnf_name2docker_name = dict()
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200118 self.sap_identifiers = set()
peusterm6b5224d2016-07-20 13:20:31 +0200119 # lets generate a set of subnet configurations used for e-line chaining setup
120 self.eline_subnets_src = generate_subnet_strings(50, start=200, subnet_size=24, ip=1)
121 self.eline_subnets_dst = generate_subnet_strings(50, start=200, subnet_size=24, ip=2)
peusterme26487b2016-03-08 14:00:21 +0100122
peusterm786cd542016-03-14 14:12:17 +0100123 def onboard(self):
124 """
125 Do all steps to prepare this service to be instantiated
126 :return:
127 """
128 # 1. extract the contents of the package and store them in our catalog
129 self._unpack_service_package()
130 # 2. read in all descriptor files
peusterm7ec665d2016-03-14 15:20:44 +0100131 self._load_package_descriptor()
132 self._load_nsd()
133 self._load_vnfd()
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200134 if DEPLOY_SAP:
135 self._load_saps()
peusterm786cd542016-03-14 14:12:17 +0100136 # 3. prepare container images (e.g. download or build Dockerfile)
peusterm82d406e2016-05-02 20:52:06 +0200137 if BUILD_DOCKERFILE:
138 self._load_docker_files()
139 self._build_images_from_dockerfiles()
140 else:
141 self._load_docker_urls()
142 self._pull_predefined_dockerimages()
peusterm3bb86bf2016-08-15 09:47:57 +0200143 LOG.info("On-boarded service: %r" % self.manifest.get("name"))
peusterm7ec665d2016-03-14 15:20:44 +0100144
peusterm082378b2016-03-16 20:14:22 +0100145 def start_service(self):
peusterm3444ae42016-03-16 20:46:41 +0100146 """
147 This methods creates and starts a new service instance.
148 It computes placements, iterates over all VNFDs, and starts
149 each VNFD as a Docker container in the data center selected
150 by the placement algorithm.
151 :return:
152 """
153 LOG.info("Starting service %r" % self.uuid)
stevenvanrossemd87fe472016-05-11 11:34:34 +0200154
peusterm3444ae42016-03-16 20:46:41 +0100155 # 1. each service instance gets a new uuid to identify it
peusterm082378b2016-03-16 20:14:22 +0100156 instance_uuid = str(uuid.uuid4())
peusterm3444ae42016-03-16 20:46:41 +0100157 # build a instances dict (a bit like a NSR :))
158 self.instances[instance_uuid] = dict()
159 self.instances[instance_uuid]["vnf_instances"] = list()
stevenvanrossemd87fe472016-05-11 11:34:34 +0200160
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200161 # 2. Configure the chaining of the network functions (currently only E-Line and E-LAN links supported)
162 vnf_id2vnf_name = defaultdict(lambda: "NotExistingNode",
163 reduce(lambda x, y: dict(x, **y),
164 map(lambda d: {d["vnf_id"]: d["vnf_name"]},
165 self.nsd["network_functions"])))
166
167 # 3. compute placement of this service instance (adds DC names to VNFDs)
peusterm398cd3b2016-03-21 15:04:54 +0100168 if not GK_STANDALONE_MODE:
peustermf6459542016-08-31 19:00:17 +0200169 #self._calculate_placement(FirstDcPlacement)
170 self._calculate_placement(RoundRobinDcPlacement)
peusterm3444ae42016-03-16 20:46:41 +0100171 # iterate over all vnfds that we have to start
peusterm082378b2016-03-16 20:14:22 +0100172 for vnfd in self.vnfds.itervalues():
peusterm398cd3b2016-03-21 15:04:54 +0100173 vnfi = None
174 if not GK_STANDALONE_MODE:
175 vnfi = self._start_vnfd(vnfd)
176 self.instances[instance_uuid]["vnf_instances"].append(vnfi)
stevenvanrossemd87fe472016-05-11 11:34:34 +0200177
stevenvanrossemd87fe472016-05-11 11:34:34 +0200178 vlinks = self.nsd["virtual_links"]
179 fwd_links = self.nsd["forwarding_graphs"][0]["constituent_virtual_links"]
180 eline_fwd_links = [l for l in vlinks if (l["id"] in fwd_links) and (l["connectivity_type"] == "E-Line")]
stevenvanrossem6d5019a2016-08-12 23:00:22 +0200181 elan_fwd_links = [l for l in vlinks if (l["id"] in fwd_links) and (l["connectivity_type"] == "E-LAN")]
stevenvanrossemd87fe472016-05-11 11:34:34 +0200182
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200183 # 4a. deploy E-Line links
stevenvanrossemaa6d3a72016-08-10 13:23:24 +0200184 # cookie is used as identifier for the flowrules installed by the dummygatekeeper
185 # eg. different services get a unique cookie for their flowrules
186 cookie = 1
stevenvanrossemd87fe472016-05-11 11:34:34 +0200187 for link in eline_fwd_links:
peusterm6b5224d2016-07-20 13:20:31 +0200188 src_id, src_if_name = link["connection_points_reference"][0].split(":")
189 dst_id, dst_if_name = link["connection_points_reference"][1].split(":")
stevenvanrossemd87fe472016-05-11 11:34:34 +0200190
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200191 # check if there is a SAP in the link
192 if src_id in self.sap_identifiers:
193 src_docker_name = "{0}_{1}".format(src_id, src_if_name)
194 src_id = src_docker_name
195 else:
196 src_docker_name = src_id
197
198 if dst_id in self.sap_identifiers:
199 dst_docker_name = "{0}_{1}".format(dst_id, dst_if_name)
200 dst_id = dst_docker_name
201 else:
202 dst_docker_name = dst_id
203
peusterm6b5224d2016-07-20 13:20:31 +0200204 src_name = vnf_id2vnf_name[src_id]
205 dst_name = vnf_id2vnf_name[dst_id]
peusterm9fb74ec2016-06-16 11:30:55 +0200206
peusterm6b5224d2016-07-20 13:20:31 +0200207 LOG.debug(
208 "Setting up E-Line link. %s(%s:%s) -> %s(%s:%s)" % (
209 src_name, src_id, src_if_name, dst_name, dst_id, dst_if_name))
210
211 if (src_name in self.vnfds) and (dst_name in self.vnfds):
212 network = self.vnfds[src_name].get("dc").net # there should be a cleaner way to find the DCNetwork
peusterm6b5224d2016-07-20 13:20:31 +0200213 LOG.debug(src_docker_name)
214 ret = network.setChain(
215 src_docker_name, dst_docker_name,
216 vnf_src_interface=src_if_name, vnf_dst_interface=dst_if_name,
peusterm76eb8652016-09-06 11:07:16 +0200217 bidirectional=BIDIRECTIONAL_CHAIN, cmd="add-flow", cookie=cookie, priority=10)
stevenvanrossemd87fe472016-05-11 11:34:34 +0200218
peusterm6b5224d2016-07-20 13:20:31 +0200219 # re-configure the VNFs IP assignment and ensure that a new subnet is used for each E-Link
220 src_vnfi = self._get_vnf_instance(instance_uuid, src_name)
221 if src_vnfi is not None:
222 self._vnf_reconfigure_network(src_vnfi, src_if_name, self.eline_subnets_src.pop(0))
223 dst_vnfi = self._get_vnf_instance(instance_uuid, dst_name)
224 if dst_vnfi is not None:
225 self._vnf_reconfigure_network(dst_vnfi, dst_if_name, self.eline_subnets_dst.pop(0))
226
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200227 # 4b. deploy E-LAN links
stevenvanrossem6d5019a2016-08-12 23:00:22 +0200228 base = 10
229 for link in elan_fwd_links:
230 # generate lan ip address
231 ip = 1
232 for intf in link["connection_points_reference"]:
233 ip_address = generate_lan_string("10.0", base, subnet_size=24, ip=ip)
234 vnf_id, intf_name = intf.split(":")
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200235 if vnf_id in self.sap_identifiers:
236 src_docker_name = "{0}_{1}".format(vnf_id, intf_name)
237 vnf_id = src_docker_name
stevenvanrossem6d5019a2016-08-12 23:00:22 +0200238 vnf_name = vnf_id2vnf_name[vnf_id]
239 LOG.debug(
240 "Setting up E-LAN link. %s(%s:%s) -> %s" % (
241 vnf_name, vnf_id, intf_name, ip_address))
242
243 if vnf_name in self.vnfds:
244 # re-configure the VNFs IP assignment and ensure that a new subnet is used for each E-LAN
peustermb1cf5372016-08-23 14:02:09 +0200245 # E-LAN relies on the learning switch capability of Ryu which has to be turned on in the topology
246 # (DCNetwork(controller=RemoteController, enable_learning=True)), so no explicit chaining is necessary.
stevenvanrossem6d5019a2016-08-12 23:00:22 +0200247 vnfi = self._get_vnf_instance(instance_uuid, vnf_name)
248 if vnfi is not None:
249 self._vnf_reconfigure_network(vnfi, intf_name, ip_address)
250 # increase for the next ip address on this E-LAN
251 ip += 1
252 # increase the base ip address for the next E-LAN
253 base += 1
254
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200255 # 5. run the emulator specific entrypoint scripts in the VNFIs of this service instance
peusterm8484b902016-06-21 09:03:35 +0200256 self._trigger_emulator_start_scripts_in_vnfis(self.instances[instance_uuid]["vnf_instances"])
257
peusterm3444ae42016-03-16 20:46:41 +0100258 LOG.info("Service started. Instance id: %r" % instance_uuid)
peusterm082378b2016-03-16 20:14:22 +0100259 return instance_uuid
260
edmaas9c4fd112016-10-05 19:45:57 +0200261 def stop_service(self, instance_uuid):
edmaasd454d542016-09-29 13:19:22 +0200262 """
263 This method stops a running service instance.
edmaas74d72492016-10-05 19:59:22 +0200264 It iterates over all VNF instances, stopping them each
edmaasd454d542016-09-29 13:19:22 +0200265 and removing them from their data center.
266
edmaas74d72492016-10-05 19:59:22 +0200267 :param instance_uuid: the uuid of the service instance to be stopped
edmaasd454d542016-09-29 13:19:22 +0200268 """
edmaas9c4fd112016-10-05 19:45:57 +0200269 LOG.info("Stopping service %r" % self.uuid)
270 # get relevant information
271 # instance_uuid = str(self.uuid.uuid4())
272 vnf_instances = self.instances[instance_uuid]["vnf_instances"]
273
274 for v in vnf_instances:
edmaas74d72492016-10-05 19:59:22 +0200275 self._stop_vnfi(v)
edmaas9c4fd112016-10-05 19:45:57 +0200276
277 if not GK_STANDALONE_MODE:
278 # remove placement?
279 # self._remove_placement(RoundRobinPlacement)
280 None
281
282 # last step: remove the instance from the list of all instances
283 del self.instances[instance_uuid]
edmaasd454d542016-09-29 13:19:22 +0200284
peusterm398cd3b2016-03-21 15:04:54 +0100285 def _start_vnfd(self, vnfd):
286 """
287 Start a single VNFD of this service
288 :param vnfd: vnfd descriptor dict
289 :return:
290 """
291 # iterate over all deployment units within each VNFDs
292 for u in vnfd.get("virtual_deployment_units"):
293 # 1. get the name of the docker image to start and the assigned DC
peusterm56356cb2016-05-03 10:43:43 +0200294 vnf_name = vnfd.get("name")
295 if vnf_name not in self.remote_docker_image_urls:
296 raise Exception("No image name for %r found. Abort." % vnf_name)
297 docker_name = self.remote_docker_image_urls.get(vnf_name)
peusterm398cd3b2016-03-21 15:04:54 +0100298 target_dc = vnfd.get("dc")
299 # 2. perform some checks to ensure we can start the container
300 assert(docker_name is not None)
301 assert(target_dc is not None)
302 if not self._check_docker_image_exists(docker_name):
303 raise Exception("Docker image %r not found. Abort." % docker_name)
304 # 3. do the dc.startCompute(name="foobar") call to run the container
305 # TODO consider flavors, and other annotations
stevenvanrossemd87fe472016-05-11 11:34:34 +0200306 intfs = vnfd.get("connection_points")
stevenvanrossemeae73082016-08-05 16:22:12 +0200307
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200308 # TODO: get all vnf id's from the nsd for this vnfd and use those as dockername
stevenvanrossem11a021f2016-08-05 13:43:00 +0200309 # use the vnf_id in the nsd as docker name
310 # so deployed containers can be easily mapped back to the nsd
311 vnf_name2id = defaultdict(lambda: "NotExistingNode",
312 reduce(lambda x, y: dict(x, **y),
313 map(lambda d: {d["vnf_name"]: d["vnf_id"]},
314 self.nsd["network_functions"])))
315 self.vnf_name2docker_name[vnf_name] = vnf_name2id[vnf_name]
316 # self.vnf_name2docker_name[vnf_name] = GK.get_next_vnf_name()
317
peusterm6b5224d2016-07-20 13:20:31 +0200318 LOG.info("Starting %r as %r in DC %r" % (vnf_name, self.vnf_name2docker_name[vnf_name], vnfd.get("dc")))
peusterm761c14d2016-07-19 09:31:19 +0200319 LOG.debug("Interfaces for %r: %r" % (vnf_name, intfs))
peusterm6b5224d2016-07-20 13:20:31 +0200320 vnfi = target_dc.startCompute(self.vnf_name2docker_name[vnf_name], network=intfs, image=docker_name, flavor_name="small")
peusterm398cd3b2016-03-21 15:04:54 +0100321 return vnfi
322
edmaas74d72492016-10-05 19:59:22 +0200323 def _stop_vnfi(self, vnfi):
edmaasd454d542016-09-29 13:19:22 +0200324 """
edmaas74d72492016-10-05 19:59:22 +0200325 Stop a VNF instance.
edmaasd454d542016-09-29 13:19:22 +0200326
edmaas74d72492016-10-05 19:59:22 +0200327 :param vnfi: vnf instance to be stopped
edmaasd454d542016-09-29 13:19:22 +0200328 """
edmaas9c4fd112016-10-05 19:45:57 +0200329 # Find the correct datacenter
330 status = vnfi.getStatus()
331 dc = vnfi.datacenter
332 # stop the vnfi
edmaas74d72492016-10-05 19:59:22 +0200333 LOG.info("Stopping the vnf instance contained in %r in DC %r" % (status["name"], dc))
edmaas9c4fd112016-10-05 19:45:57 +0200334 dc.stopCompute(status["name"])
edmaasd454d542016-09-29 13:19:22 +0200335
peusterm6b5224d2016-07-20 13:20:31 +0200336 def _get_vnf_instance(self, instance_uuid, name):
337 """
338 Returns the Docker object for the given VNF name (or Docker name).
339 :param instance_uuid: UUID of the service instance to search in.
340 :param name: VNF name or Docker name. We are fuzzy here.
341 :return:
342 """
343 dn = name
344 if name in self.vnf_name2docker_name:
345 dn = self.vnf_name2docker_name[name]
346 for vnfi in self.instances[instance_uuid]["vnf_instances"]:
347 if vnfi.name == dn:
348 return vnfi
349 LOG.warning("No container with name: %r found.")
350 return None
351
352 @staticmethod
353 def _vnf_reconfigure_network(vnfi, if_name, net_str):
354 """
355 Reconfigure the network configuration of a specific interface
356 of a running container.
357 :param vnfi: container instacne
358 :param if_name: interface name
359 :param net_str: network configuration string, e.g., 1.2.3.4/24
360 :return:
361 """
362 intf = vnfi.intf(intf=if_name)
363 if intf is not None:
364 intf.setIP(net_str)
365 LOG.debug("Reconfigured network of %s:%s to %r" % (vnfi.name, if_name, net_str))
366 else:
367 LOG.warning("Interface not found: %s:%s. Network reconfiguration skipped." % (vnfi.name, if_name))
368
369
peusterm8484b902016-06-21 09:03:35 +0200370 def _trigger_emulator_start_scripts_in_vnfis(self, vnfi_list):
371 for vnfi in vnfi_list:
372 config = vnfi.dcinfo.get("Config", dict())
373 env = config.get("Env", list())
374 for env_var in env:
375 if "SON_EMU_CMD=" in env_var:
376 cmd = str(env_var.split("=")[1])
peusterme66edf72016-08-23 11:11:12 +0200377 LOG.info("Executing entry point script in %r: %r" % (vnfi.name, cmd))
378 # execute command in new thread to ensure that GK is not blocked by VNF
379 t = threading.Thread(target=vnfi.cmdPrint, args=(cmd,))
380 t.daemon = True
381 t.start()
peusterm8484b902016-06-21 09:03:35 +0200382
peusterm786cd542016-03-14 14:12:17 +0100383 def _unpack_service_package(self):
384 """
385 unzip *.son file and store contents in CATALOG_FOLDER/services/<service_uuid>/
386 """
peusterm82d406e2016-05-02 20:52:06 +0200387 LOG.info("Unzipping: %r" % self.package_file_path)
peusterm786cd542016-03-14 14:12:17 +0100388 with zipfile.ZipFile(self.package_file_path, "r") as z:
389 z.extractall(self.package_content_path)
390
peusterm82d406e2016-05-02 20:52:06 +0200391
peusterm7ec665d2016-03-14 15:20:44 +0100392 def _load_package_descriptor(self):
393 """
394 Load the main package descriptor YAML and keep it as dict.
395 :return:
396 """
397 self.manifest = load_yaml(
398 os.path.join(
399 self.package_content_path, "META-INF/MANIFEST.MF"))
400
401 def _load_nsd(self):
402 """
403 Load the entry NSD YAML and keep it as dict.
404 :return:
405 """
406 if "entry_service_template" in self.manifest:
407 nsd_path = os.path.join(
408 self.package_content_path,
409 make_relative_path(self.manifest.get("entry_service_template")))
410 self.nsd = load_yaml(nsd_path)
peusterm757fe9a2016-04-04 14:11:58 +0200411 LOG.debug("Loaded NSD: %r" % self.nsd.get("name"))
peusterm7ec665d2016-03-14 15:20:44 +0100412
413 def _load_vnfd(self):
414 """
415 Load all VNFD YAML files referenced in MANIFEST.MF and keep them in dict.
416 :return:
417 """
418 if "package_content" in self.manifest:
419 for pc in self.manifest.get("package_content"):
420 if pc.get("content-type") == "application/sonata.function_descriptor":
421 vnfd_path = os.path.join(
422 self.package_content_path,
423 make_relative_path(pc.get("name")))
424 vnfd = load_yaml(vnfd_path)
peusterm757fe9a2016-04-04 14:11:58 +0200425 self.vnfds[vnfd.get("name")] = vnfd
426 LOG.debug("Loaded VNFD: %r" % vnfd.get("name"))
peusterm7ec665d2016-03-14 15:20:44 +0100427
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200428 def _load_saps(self):
429 # Each Service Access Point (connection_point) in the nsd is getting its own container
430 SAPs = [p["id"] for p in self.nsd["connection_points"] if p["type"] == "interface"]
431 for sap in SAPs:
432 # endpoints needed in this service
433 sap_vnf_id, sap_vnf_interface = sap.split(':')
434 # set of the connection_point ids found in the nsd (in the examples this is 'ns')
435 self.sap_identifiers.add(sap_vnf_id)
436
peusterm76eb8652016-09-06 11:07:16 +0200437 sap_docker_name = "%s_%s" % (sap_vnf_id, sap_vnf_interface)
stevenvanrossemdb2f9432016-08-20 00:01:11 +0200438
439 # add SAP to self.vnfds
440 sapfile = pkg_resources.resource_filename(__name__, "sap_vnfd.yml")
441 sap_vnfd = load_yaml(sapfile)
442 sap_vnfd["connection_points"][0]["id"] = sap_vnf_interface
443 sap_vnfd["name"] = sap_docker_name
444 self.vnfds[sap_docker_name] = sap_vnfd
445 # add SAP vnf to list in the NSD so it is deployed later on
446 # each SAP get a unique VNFD and vnf_id in the NSD
447 self.nsd["network_functions"].append({"vnf_id": sap_docker_name, "vnf_name": sap_docker_name})
448 LOG.debug("Loaded SAP: %r" % sap_vnfd.get("name"))
449
peusterm7ec665d2016-03-14 15:20:44 +0100450 def _load_docker_files(self):
451 """
peusterm9d7d4b02016-03-23 19:56:44 +0100452 Get all paths to Dockerfiles from VNFDs and store them in dict.
peusterm7ec665d2016-03-14 15:20:44 +0100453 :return:
454 """
peusterm9d7d4b02016-03-23 19:56:44 +0100455 for k, v in self.vnfds.iteritems():
456 for vu in v.get("virtual_deployment_units"):
457 if vu.get("vm_image_format") == "docker":
458 vm_image = vu.get("vm_image")
peusterm7ec665d2016-03-14 15:20:44 +0100459 docker_path = os.path.join(
460 self.package_content_path,
peusterm9d7d4b02016-03-23 19:56:44 +0100461 make_relative_path(vm_image))
462 self.local_docker_files[k] = docker_path
peusterm56356cb2016-05-03 10:43:43 +0200463 LOG.debug("Found Dockerfile (%r): %r" % (k, docker_path))
peusterm7ec665d2016-03-14 15:20:44 +0100464
peusterm82d406e2016-05-02 20:52:06 +0200465 def _load_docker_urls(self):
466 """
467 Get all URLs to pre-build docker images in some repo.
468 :return:
469 """
470 for k, v in self.vnfds.iteritems():
471 for vu in v.get("virtual_deployment_units"):
472 if vu.get("vm_image_format") == "docker":
peusterm35ba4052016-05-02 21:21:14 +0200473 url = vu.get("vm_image")
474 if url is not None:
475 url = url.replace("http://", "")
476 self.remote_docker_image_urls[k] = url
peusterm56356cb2016-05-03 10:43:43 +0200477 LOG.debug("Found Docker image URL (%r): %r" % (k, self.remote_docker_image_urls[k]))
peusterm82d406e2016-05-02 20:52:06 +0200478
peustermbdfab7e2016-03-14 16:03:30 +0100479 def _build_images_from_dockerfiles(self):
480 """
481 Build Docker images for each local Dockerfile found in the package: self.local_docker_files
482 """
peusterm398cd3b2016-03-21 15:04:54 +0100483 if GK_STANDALONE_MODE:
484 return # do not build anything in standalone mode
peustermbdfab7e2016-03-14 16:03:30 +0100485 dc = DockerClient()
486 LOG.info("Building %d Docker images (this may take several minutes) ..." % len(self.local_docker_files))
487 for k, v in self.local_docker_files.iteritems():
488 for line in dc.build(path=v.replace("Dockerfile", ""), tag=k, rm=False, nocache=False):
489 LOG.debug("DOCKER BUILD: %s" % line)
490 LOG.info("Docker image created: %s" % k)
491
peusterm82d406e2016-05-02 20:52:06 +0200492 def _pull_predefined_dockerimages(self):
peustermbdfab7e2016-03-14 16:03:30 +0100493 """
494 If the package contains URLs to pre-build Docker images, we download them with this method.
495 """
peusterm35ba4052016-05-02 21:21:14 +0200496 dc = DockerClient()
497 for url in self.remote_docker_image_urls.itervalues():
peusterm56356cb2016-05-03 10:43:43 +0200498 if not FORCE_PULL: # only pull if not present (speedup for development)
499 if len(dc.images(name=url)) > 0:
500 LOG.debug("Image %r present. Skipping pull." % url)
501 continue
peusterm35ba4052016-05-02 21:21:14 +0200502 LOG.info("Pulling image: %r" % url)
503 dc.pull(url,
504 insecure_registry=True)
peusterm786cd542016-03-14 14:12:17 +0100505
peusterm3444ae42016-03-16 20:46:41 +0100506 def _check_docker_image_exists(self, image_name):
peusterm3f307142016-03-16 21:02:53 +0100507 """
508 Query the docker service and check if the given image exists
509 :param image_name: name of the docker image
510 :return:
511 """
512 return len(DockerClient().images(image_name)) > 0
peusterm3444ae42016-03-16 20:46:41 +0100513
peusterm082378b2016-03-16 20:14:22 +0100514 def _calculate_placement(self, algorithm):
515 """
516 Do placement by adding the a field "dc" to
517 each VNFD that points to one of our
518 data center objects known to the gatekeeper.
519 """
520 assert(len(self.vnfds) > 0)
521 assert(len(GK.dcs) > 0)
522 # instantiate algorithm an place
523 p = algorithm()
524 p.place(self.nsd, self.vnfds, GK.dcs)
525 LOG.info("Using placement algorithm: %r" % p.__class__.__name__)
526 # lets print the placement result
527 for name, vnfd in self.vnfds.iteritems():
528 LOG.info("Placed VNF %r on DC %r" % (name, str(vnfd.get("dc"))))
529
530
531"""
532Some (simple) placement algorithms
533"""
534
535
536class FirstDcPlacement(object):
537 """
538 Placement: Always use one and the same data center from the GK.dcs dict.
539 """
540 def place(self, nsd, vnfds, dcs):
541 for name, vnfd in vnfds.iteritems():
542 vnfd["dc"] = list(dcs.itervalues())[0]
543
peusterme26487b2016-03-08 14:00:21 +0100544
peustermf6459542016-08-31 19:00:17 +0200545class RoundRobinDcPlacement(object):
546 """
547 Placement: Distribute VNFs across all available DCs in a round robin fashion.
548 """
peustermf6459542016-08-31 19:00:17 +0200549 def place(self, nsd, vnfds, dcs):
550 c = 0
edmaasd454d542016-09-29 13:19:22 +0200551 dcs_list = list(dcs.itervalues())
peustermf6459542016-08-31 19:00:17 +0200552 for name, vnfd in vnfds.iteritems():
553 vnfd["dc"] = dcs_list[c % len(dcs_list)]
554 c += 1 # inc. c to use next DC
555
556
557
558
peusterme26487b2016-03-08 14:00:21 +0100559"""
560Resource definitions and API endpoints
561"""
562
563
564class Packages(fr.Resource):
565
566 def post(self):
567 """
peusterm26455852016-03-08 14:23:53 +0100568 Upload a *.son service package to the dummy gatekeeper.
569
peusterme26487b2016-03-08 14:00:21 +0100570 We expect request with a *.son file and store it in UPLOAD_FOLDER
peusterm26455852016-03-08 14:23:53 +0100571 :return: UUID
peusterme26487b2016-03-08 14:00:21 +0100572 """
573 try:
574 # get file contents
wtavernib8d9ecb2016-03-25 15:18:31 +0100575 print(request.files)
peusterm593ca582016-03-30 19:55:01 +0200576 # lets search for the package in the request
577 if "package" in request.files:
578 son_file = request.files["package"]
579 # elif "file" in request.files:
580 # son_file = request.files["file"]
581 else:
582 return {"service_uuid": None, "size": 0, "sha1": None, "error": "upload failed. file not found."}, 500
peusterme26487b2016-03-08 14:00:21 +0100583 # generate a uuid to reference this package
584 service_uuid = str(uuid.uuid4())
peusterm786cd542016-03-14 14:12:17 +0100585 file_hash = hashlib.sha1(str(son_file)).hexdigest()
peusterme26487b2016-03-08 14:00:21 +0100586 # ensure that upload folder exists
587 ensure_dir(UPLOAD_FOLDER)
588 upload_path = os.path.join(UPLOAD_FOLDER, "%s.son" % service_uuid)
589 # store *.son file to disk
peusterm786cd542016-03-14 14:12:17 +0100590 son_file.save(upload_path)
peusterme26487b2016-03-08 14:00:21 +0100591 size = os.path.getsize(upload_path)
peusterm786cd542016-03-14 14:12:17 +0100592 # create a service object and register it
593 s = Service(service_uuid, file_hash, upload_path)
594 GK.register_service_package(service_uuid, s)
peusterme26487b2016-03-08 14:00:21 +0100595 # generate the JSON result
peusterm938143e2016-09-15 15:39:36 +0200596 return {"service_uuid": service_uuid, "size": size, "sha1": file_hash, "error": None}, 201
peusterme26487b2016-03-08 14:00:21 +0100597 except Exception as ex:
peusterm786cd542016-03-14 14:12:17 +0100598 LOG.exception("Service package upload failed:")
peusterm593ca582016-03-30 19:55:01 +0200599 return {"service_uuid": None, "size": 0, "sha1": None, "error": "upload failed"}, 500
peusterme26487b2016-03-08 14:00:21 +0100600
601 def get(self):
peusterm26455852016-03-08 14:23:53 +0100602 """
603 Return a list of UUID's of uploaded service packages.
604 :return: dict/list
605 """
peusterm075b46a2016-07-20 17:08:00 +0200606 LOG.info("GET /packages")
peusterm786cd542016-03-14 14:12:17 +0100607 return {"service_uuid_list": list(GK.services.iterkeys())}
peusterme26487b2016-03-08 14:00:21 +0100608
609
610class Instantiations(fr.Resource):
611
612 def post(self):
peusterm26455852016-03-08 14:23:53 +0100613 """
614 Instantiate a service specified by its UUID.
615 Will return a new UUID to identify the running service instance.
616 :return: UUID
617 """
peusterm64b45502016-03-16 21:15:14 +0100618 # try to extract the service uuid from the request
peusterm26455852016-03-08 14:23:53 +0100619 json_data = request.get_json(force=True)
peusterm64b45502016-03-16 21:15:14 +0100620 service_uuid = json_data.get("service_uuid")
621
622 # lets be a bit fuzzy here to make testing easier
623 if service_uuid is None and len(GK.services) > 0:
624 # if we don't get a service uuid, we simple start the first service in the list
625 service_uuid = list(GK.services.iterkeys())[0]
626
peustermbea87372016-03-16 19:37:35 +0100627 if service_uuid in GK.services:
peusterm64b45502016-03-16 21:15:14 +0100628 # ok, we have a service uuid, lets start the service
peustermbea87372016-03-16 19:37:35 +0100629 service_instance_uuid = GK.services.get(service_uuid).start_service()
edmaas59b28fc2016-11-01 17:11:47 +0100630 return {"service_instance_uuid": service_instance_uuid}, 201
peustermbea87372016-03-16 19:37:35 +0100631 return "Service not found", 404
peusterme26487b2016-03-08 14:00:21 +0100632
633 def get(self):
peusterm26455852016-03-08 14:23:53 +0100634 """
635 Returns a list of UUIDs containing all running services.
636 :return: dict / list
637 """
peusterm075b46a2016-07-20 17:08:00 +0200638 LOG.info("GET /instantiations")
639 return {"service_instantiations_list": [
peusterm64b45502016-03-16 21:15:14 +0100640 list(s.instances.iterkeys()) for s in GK.services.itervalues()]}
peusterm786cd542016-03-14 14:12:17 +0100641
edmaasd454d542016-09-29 13:19:22 +0200642 def delete(self):
643 """
edmaas74d72492016-10-05 19:59:22 +0200644 Stops a running service specified by its service and instance UUID.
edmaasd454d542016-09-29 13:19:22 +0200645 """
edmaas74d72492016-10-05 19:59:22 +0200646 # try to extract the service and instance UUID from the request
edmaasd454d542016-09-29 13:19:22 +0200647 json_data = request.get_json(force=True)
648 service_uuid = json_data.get("service_uuid")
edmaas59b28fc2016-11-01 17:11:47 +0100649 instance_uuid = json_data.get("service_instance_uuid")
edmaas9c4fd112016-10-05 19:45:57 +0200650
651 # try to be fuzzy
652 if service_uuid is None and len(GK.services) > 0:
653 #if we don't get a service uuid, we simply stop the last service in the list
654 service_uuid = list(GK.services.iterkeys())[0]
655 if instance_uuid is None and len(GK.services[service_uuid].instances) > 0:
656 instance_uuid = list(GK.services[service_uuid].instances.iterkeys())[0]
edmaasd454d542016-09-29 13:19:22 +0200657
edmaas74d72492016-10-05 19:59:22 +0200658 if service_uuid in GK.services and instance_uuid in GK.services[service_uuid].instances:
659 # valid service and instance UUID, stop service
edmaas9c4fd112016-10-05 19:45:57 +0200660 GK.services.get(service_uuid).stop_service(instance_uuid)
edmaas59b28fc2016-11-01 17:11:47 +0100661 del GK.services.get(service_uuid).instances[instance_uuid]
662 return
edmaasd454d542016-09-29 13:19:22 +0200663 return "Service not found", 404
664
edmaas74d72492016-10-05 19:59:22 +0200665class Exit(fr.Resource):
edmaas9c4fd112016-10-05 19:45:57 +0200666
667 def put(self):
668 """
669 Stop the running Containernet instance regardless of data transmitted
670 """
edmaas59b28fc2016-11-01 17:11:47 +0100671 GK.net.stop()
672
673
674def initialize_GK():
675 global GK
676 GK = Gatekeeper()
677
edmaas9c4fd112016-10-05 19:45:57 +0200678
peusterme26487b2016-03-08 14:00:21 +0100679
680# create a single, global GK object
edmaas59b28fc2016-11-01 17:11:47 +0100681GK = None
682initialize_GK()
peusterme26487b2016-03-08 14:00:21 +0100683# setup Flask
684app = Flask(__name__)
685app.config['MAX_CONTENT_LENGTH'] = 512 * 1024 * 1024 # 512 MB max upload
686api = fr.Api(app)
687# define endpoints
peusterm593ca582016-03-30 19:55:01 +0200688api.add_resource(Packages, '/packages')
689api.add_resource(Instantiations, '/instantiations')
edmaas74d72492016-10-05 19:59:22 +0200690api.add_resource(Exit, '/emulator/exit')
peusterme26487b2016-03-08 14:00:21 +0100691
692
edmaas59b28fc2016-11-01 17:11:47 +0100693#def initialize_GK():
694# global GK
695# GK = Gatekeeper()
696
697
peusterm082378b2016-03-16 20:14:22 +0100698def start_rest_api(host, port, datacenters=dict()):
peustermbea87372016-03-16 19:37:35 +0100699 GK.dcs = datacenters
peusterme26487b2016-03-08 14:00:21 +0100700 # start the Flask server (not the best performance but ok for our use case)
701 app.run(host=host,
702 port=port,
703 debug=True,
704 use_reloader=False # this is needed to run Flask in a non-main thread
705 )
706
707
708def ensure_dir(name):
709 if not os.path.exists(name):
peusterm7ec665d2016-03-14 15:20:44 +0100710 os.makedirs(name)
711
712
713def load_yaml(path):
714 with open(path, "r") as f:
715 try:
716 r = yaml.load(f)
717 except yaml.YAMLError as exc:
718 LOG.exception("YAML parse error")
719 r = dict()
720 return r
721
722
723def make_relative_path(path):
peusterm9d7d4b02016-03-23 19:56:44 +0100724 if path.startswith("file://"):
725 path = path.replace("file://", "", 1)
peusterm7ec665d2016-03-14 15:20:44 +0100726 if path.startswith("/"):
peusterm9d7d4b02016-03-23 19:56:44 +0100727 path = path.replace("/", "", 1)
peusterm7ec665d2016-03-14 15:20:44 +0100728 return path
729
730
stevenvanrossem6d5019a2016-08-12 23:00:22 +0200731def generate_lan_string(prefix, base, subnet_size=24, ip=0):
732 """
733 Helper to generate different network configuration strings.
734 """
735 r = "%s.%d.%d/%d" % (prefix, base, ip, subnet_size)
736 return r
737
738
peusterm6b5224d2016-07-20 13:20:31 +0200739def generate_subnet_strings(n, start=1, subnet_size=24, ip=0):
740 """
741 Helper to generate different network configuration strings.
742 """
743 r = list()
744 for i in range(start, start + n):
745 r.append("%d.0.0.%d/%d" % (i, ip, subnet_size))
746 return r
747
748
peusterme26487b2016-03-08 14:00:21 +0100749if __name__ == '__main__':
750 """
751 Lets allow to run the API in standalone mode.
752 """
peusterm398cd3b2016-03-21 15:04:54 +0100753 GK_STANDALONE_MODE = True
peusterme26487b2016-03-08 14:00:21 +0100754 logging.getLogger("werkzeug").setLevel(logging.INFO)
755 start_rest_api("0.0.0.0", 8000)
756