5GTANGO LLCM: Refactoring, cleanup 71/7071/1
authorpeusterm <manuel.peuster@uni-paderborn.de>
Wed, 19 Dec 2018 08:58:17 +0000 (09:58 +0100)
committerpeusterm <manuel.peuster@uni-paderborn.de>
Wed, 19 Dec 2018 08:58:17 +0000 (09:58 +0100)
Change-Id: I725c34723ca431c1544f64edd6e01cb70cacad85
Signed-off-by: peusterm <manuel.peuster@uni-paderborn.de>
src/emuvim/api/tango/llcm.py
src/emuvim/test/unittests/test_tango_llcm.py

index 174bcf8..d7c24dd 100755 (executable)
@@ -39,13 +39,10 @@ import threading
 from docker import DockerClient
 from flask import Flask, request
 import flask_restful as fr
-from collections import defaultdict
-import pkg_resources
 from subprocess import Popen
 import ipaddress
 import copy
 import time
-from functools import reduce
 
 
 LOG = logging.getLogger("5gtango.llcm")
@@ -66,11 +63,6 @@ GK_STANDALONE_MODE = False
 # should a new version of an image be pulled even if its available
 FORCE_PULL = False
 
-# Automatically deploy SAPs (endpoints) of the service as new containers
-# Attention: This is not a configuration switch but a global variable!
-# Don't change its default value.
-DEPLOY_SAP = False
-
 # flag to indicate if we use bidirectional forwarding rules in the
 # automatic chaining process
 BIDIRECTIONAL_CHAIN = True
@@ -86,26 +78,9 @@ AUTO_DEPLOY = False
 # and also automatically terminate any other running services
 AUTO_DELETE = False
 
-
-def generate_subnets(prefix, base, subnet_size=50, mask=24):
-    # Generate a list of ipaddress in subnets
-    r = list()
-    for net in range(base, base + subnet_size):
-        subnet = "{0}.{1}.0/{2}".format(prefix, net, mask)
-        r.append(ipaddress.ip_network(unicode(subnet)))
-    return r
-
-
-# private subnet definitions for the generated interfaces
-# 99.0.xxx.0/24
-SAP_SUBNETS = generate_subnets('99.0', 0, subnet_size=50, mask=24)
-# 30.0.xxx.0/24
-ELAN_SUBNETS = generate_subnets('30.0', 0, subnet_size=50, mask=24)
-# 20.0.xxx.0/24
-ELINE_SUBNETS = generate_subnets('20.0', 0, subnet_size=50, mask=24)
-
-# path to the VNFD for the SAP VNF that is deployed as internal SAP point
-SAP_VNFD = None
+# global subnet definitions (see reset_subnets())
+ELAN_SUBNETS = None
+ELINE_SUBNETS = None
 
 # Time in seconds to wait for vnf stop scripts to execute fully
 VNF_STOP_WAIT_TIME = 5
@@ -123,6 +98,7 @@ class Gatekeeper(object):
         self.net = None
         # used to generate short names for VNFs (Mininet limitation)
         self.vnf_counter = 0
+        reset_subnets()
         LOG.info("Initialized 5GTANGO LLCM module.")
 
     def register_service_package(self, service_uuid, service):
@@ -155,14 +131,9 @@ class Service(object):
         self.manifest = None
         self.nsd = None
         self.vnfds = dict()
-        self.saps = dict()
-        self.saps_ext = list()
-        self.saps_int = list()
         self.local_docker_files = dict()
         self.remote_docker_image_urls = dict()
         self.instances = dict()
-        # dict to find the vnf_name for any vnf id
-        self.vnf_id2vnf_name = dict()
 
     def onboard(self):
         """
@@ -179,8 +150,6 @@ class Service(object):
             raise OnBoardingException("No NSD found.")
         if len(self.vnfds) < 1:
             raise OnBoardingException("No VNFDs found.")
-        if DEPLOY_SAP:
-            self._load_saps()
         # 3. prepare container images (e.g. download or build Dockerfile)
         if BUILD_DOCKERFILE:
             self._load_docker_files()
@@ -208,9 +177,8 @@ class Service(object):
 
         # 2. compute placement of this service instance (adds DC names to
         # VNFDs)
-        if not GK_STANDALONE_MODE:
-            # self._calculate_placement(FirstDcPlacement)
-            self._calculate_placement(RoundRobinDcPlacement)
+        # self._calculate_placement(FirstDcPlacement)
+        self._calculate_placement(RoundRobinDcPlacement)
         # 3. start all vnfds that we have in the service
         for vnf_id in self.vnfds:
             vnfd = self.vnfds[vnf_id]
@@ -225,7 +193,6 @@ class Service(object):
         if "virtual_links" in self.nsd and "forwarding_graphs" in self.nsd:
             vlinks = self.nsd["virtual_links"]
             # constituent virtual links are not checked
-            # fwd_links = self.nsd["forwarding_graphs"][0]["constituent_virtual_links"]
             eline_fwd_links = [l for l in vlinks if (
                 l["connectivity_type"] == "E-Line")]
             elan_fwd_links = [l for l in vlinks if (
@@ -243,7 +210,7 @@ class Service(object):
         # service instance
         self._trigger_emulator_start_scripts_in_vnfis(
             self.instances[instance_uuid]["vnf_instances"])
-
+        # done
         LOG.info("Service started. Instance id: %r" % instance_uuid)
         return instance_uuid
 
@@ -252,29 +219,19 @@ class Service(object):
         This method stops a running service instance.
         It iterates over all VNF instances, stopping them each
         and removing them from their data center.
-
         :param instance_uuid: the uuid of the service instance to be stopped
         """
         LOG.info("Stopping service %r" % self.uuid)
         # get relevant information
         # instance_uuid = str(self.uuid.uuid4())
         vnf_instances = self.instances[instance_uuid]["vnf_instances"]
-
         # trigger stop skripts in vnf instances and wait a few seconds for
         # completion
         self._trigger_emulator_stop_scripts_in_vnfis(vnf_instances)
         time.sleep(VNF_STOP_WAIT_TIME)
-
+        # stop all vnfs
         for v in vnf_instances:
             self._stop_vnfi(v)
-
-        for sap_name in self.saps_ext:
-            ext_sap = self.saps[sap_name]
-            target_dc = ext_sap.get("dc")
-            target_dc.removeExternalSAP(sap_name)
-            LOG.info("Stopping the SAP instance: %r in DC %r" %
-                     (sap_name, target_dc))
-
         # last step: remove the instance from the list of all instances
         del self.instances[instance_uuid]
 
@@ -388,13 +345,11 @@ class Service(object):
     def _stop_vnfi(self, vnfi):
         """
         Stop a VNF instance.
-
         :param vnfi: vnf instance to be stopped
         """
         # Find the correct datacenter
         status = vnfi.getStatus()
         dc = vnfi.datacenter
-
         # stop the vnfi
         LOG.info("Stopping the vnf instance contained in %r in DC %r" %
                  (status["name"], dc))
@@ -438,7 +393,6 @@ class Service(object):
         :param net_str: network configuration string, e.g., 1.2.3.4/24
         :return:
         """
-
         # assign new ip address
         if net_str is not None:
             intf = vnfi.intf(intf=if_name)
@@ -524,11 +478,6 @@ class Service(object):
                 raise OnBoardingException("No NSD with type 'application/vnd.5gtango.nsd' found.")
             self.nsd = load_yaml(nsd_path)
             GK.net.deployed_nsds.append(self.nsd)  # TODO this seems strange (remove?)
-            # create dict to find the vnf_name for any vnf id
-            self.vnf_id2vnf_name = defaultdict(lambda: "NotExistingNode",
-                                               reduce(lambda x, y: dict(x, **y),
-                                                       map(lambda d: {d["vnf_id"]: d["vnf_name"]},
-                                                           self.nsd["network_functions"])))
             LOG.debug("Loaded NSD: %r" % self.nsd.get("name"))
         else:
             raise OnBoardingException(
@@ -540,7 +489,6 @@ class Service(object):
         Load all VNFD YAML files referenced in MANIFEST.MF and keep them in dict.
         :return:
         """
-
         # first make a list of all the vnfds in the package
         vnfd_set = dict()
         if "package_content" in self.manifest:
@@ -555,83 +503,11 @@ class Service(object):
             if len(vnfd_set) < 1:
                 raise OnBoardingException("No VNFDs found.")
             # then link each vnf_id in the nsd to its vnfd
-            for vnf_id in self.vnf_id2vnf_name:
-                vnf_name = self.vnf_id2vnf_name[vnf_id]
-                self.vnfds[vnf_id] = vnfd_set[vnf_name]
-                LOG.debug("Loaded VNFD: {0} id: {1}".format(vnf_name, vnf_id))
-
-    def _load_saps(self):
-        # create list of all SAPs
-        # check if we need to deploy management ports
-        if USE_DOCKER_MGMT:
-            SAPs = [p for p in self.nsd["connection_points"]
-                    if 'management' not in p.get('type')]
-        else:
-            SAPs = [p for p in self.nsd["connection_points"]]
-
-        for sap in SAPs:
-            # endpoint needed in this service
-            sap_id, sap_interface, sap_docker_name = parse_interface(sap['id'])
-            # make sure SAP has type set (default internal)
-            sap["type"] = sap.get("type", 'internal')
-
-            # Each Service Access Point (connection_point) in the nsd is an IP
-            # address on the host
-            if sap["type"] == "external":
-                # add to vnfds to calculate placement later on
-                sap_net = SAP_SUBNETS.pop(0)
-                self.saps[sap_docker_name] = {
-                    "name": sap_docker_name, "type": "external", "net": sap_net}
-                # add SAP vnf to list in the NSD so it is deployed later on
-                # each SAP gets a unique VNFD and vnf_id in the NSD and custom
-                # type (only defined in the dummygatekeeper)
-                self.nsd["network_functions"].append(
-                    {"vnf_id": sap_docker_name, "vnf_name": sap_docker_name, "vnf_type": "sap_ext"})
-
-            # Each Service Access Point (connection_point) in the nsd is
-            # getting its own container (default)
-            elif sap["type"] == "internal" or sap["type"] == "management":
-                # add SAP to self.vnfds
-                if SAP_VNFD is None:
-                    sapfile = pkg_resources.resource_filename(
-                        __name__, "sap_vnfd.yml")
-                else:
-                    sapfile = SAP_VNFD
-                sap_vnfd = load_yaml(sapfile)
-                sap_vnfd["connection_points"][0]["id"] = sap_interface
-                sap_vnfd["name"] = sap_docker_name
-                sap_vnfd["type"] = "internal"
-                # add to vnfds to calculate placement later on and deploy
-                self.saps[sap_docker_name] = sap_vnfd
-                # add SAP vnf to list in the NSD so it is deployed later on
-                # each SAP get a unique VNFD and vnf_id in the NSD
-                self.nsd["network_functions"].append(
-                    {"vnf_id": sap_docker_name, "vnf_name": sap_docker_name, "vnf_type": "sap_int"})
-
-            LOG.debug("Loaded SAP: name: {0}, type: {1}".format(
-                sap_docker_name, sap['type']))
-
-        # create sap lists
-        self.saps_ext = [self.saps[sap]['name']
-                         for sap in self.saps if self.saps[sap]["type"] == "external"]
-        self.saps_int = [self.saps[sap]['name']
-                         for sap in self.saps if self.saps[sap]["type"] == "internal"]
-
-    def _start_sap(self, sap, instance_uuid):
-        if not DEPLOY_SAP:
-            return
-
-        LOG.info('start SAP: {0} ,type: {1}'.format(sap['name'], sap['type']))
-        if sap["type"] == "internal":
-            vnfi = None
-            if not GK_STANDALONE_MODE:
-                vnfi = self._start_vnfd(sap, sap['name'], type='sap_int')
-            self.instances[instance_uuid]["vnf_instances"].append(vnfi)
-
-        elif sap["type"] == "external":
-            target_dc = sap.get("dc")
-            # add interface to dc switch
-            target_dc.attachExternalSAP(sap['name'], sap['net'])
+            for v in self.nsd.get("network_functions"):
+                if v.get("vnf_name") in vnfd_set:
+                    self.vnfds[v.get("vnf_id")] = vnfd_set[v.get("vnf_name")]
+                LOG.debug("Loaded VNFD: {0} id: {1}"
+                          .format(v.get("vnf_name"), v.get("vnf_id")))
 
     def _connect_elines(self, eline_fwd_links, instance_uuid):
         """
@@ -652,79 +528,35 @@ class Service(object):
                         link["connection_points_reference"]):
                     continue
 
-            src_id, src_if_name, src_sap_id = parse_interface(
+            src_id, src_if_name = parse_interface(
                 link["connection_points_reference"][0])
-            dst_id, dst_if_name, dst_sap_id = parse_interface(
+            dst_id, dst_if_name = parse_interface(
                 link["connection_points_reference"][1])
-
             setChaining = False
-            # check if there is a SAP in the link and chain everything together
-            if src_sap_id in self.saps and dst_sap_id in self.saps:
-                LOG.info(
-                    '2 SAPs cannot be chained together : {0} - {1}'.format(src_sap_id, dst_sap_id))
-                continue
-
-            elif src_sap_id in self.saps_ext:
-                src_id = src_sap_id
-                # set intf name to None so the chaining function will choose
-                # the first one
-                src_if_name = None
-                dst_vnfi = self._get_vnf_instance(instance_uuid, dst_id)
-                if dst_vnfi is not None:
-                    # choose first ip address in sap subnet
-                    sap_net = self.saps[src_sap_id]['net']
-                    sap_ip = "{0}/{1}".format(str(sap_net[2]),
-                                              sap_net.prefixlen)
-                    self._vnf_reconfigure_network(
-                        dst_vnfi, dst_if_name, sap_ip)
-                    setChaining = True
-
-            elif dst_sap_id in self.saps_ext:
-                dst_id = dst_sap_id
-                # set intf name to None so the chaining function will choose
-                # the first one
-                dst_if_name = None
-                src_vnfi = self._get_vnf_instance(instance_uuid, src_id)
-                if src_vnfi is not None:
-                    sap_net = self.saps[dst_sap_id]['net']
-                    sap_ip = "{0}/{1}".format(str(sap_net[2]),
-                                              sap_net.prefixlen)
-                    self._vnf_reconfigure_network(
-                        src_vnfi, src_if_name, sap_ip)
-                    setChaining = True
-
-            # Link between 2 VNFs
-            else:
-                LOG.info("Creating E-Line: src={}, dst={}"
-                         .format(src_id, dst_id))
-                # make sure we use the correct sap vnf name
-                if src_sap_id in self.saps_int:
-                    src_id = src_sap_id
-                if dst_sap_id in self.saps_int:
-                    dst_id = dst_sap_id
-                # get involved vnfis
-                src_vnfi = self._get_vnf_instance(instance_uuid, src_id)
-                dst_vnfi = self._get_vnf_instance(instance_uuid, dst_id)
-
-                if src_vnfi is not None and dst_vnfi is not None:
-                    setChaining = True
-                    # re-configure the VNFs IP assignment and ensure that a new
-                    # subnet is used for each E-Link
-                    eline_net = ELINE_SUBNETS.pop(0)
-                    ip1 = "{0}/{1}".format(str(eline_net[1]),
-                                           eline_net.prefixlen)
-                    ip2 = "{0}/{1}".format(str(eline_net[2]),
-                                           eline_net.prefixlen)
-                    # check if VNFs have fixed IPs (address field in VNFDs)
-                    if (self._get_vnfd_cp_from_vnfi(src_vnfi, src_if_name)
-                            .get("address") is None):
-                        self._vnf_reconfigure_network(src_vnfi, src_if_name, ip1)
-                    # check if VNFs have fixed IPs (address field in VNFDs)
-                    if (self._get_vnfd_cp_from_vnfi(dst_vnfi, dst_if_name)
-                            .get("address") is None):
-                        self._vnf_reconfigure_network(dst_vnfi, dst_if_name, ip2)
-
-            # Set the chaining
+            LOG.info("Creating E-Line: src={}, dst={}"
+                     .format(src_id, dst_id))
+            # get involved vnfis
+            src_vnfi = self._get_vnf_instance(instance_uuid, src_id)
+            dst_vnfi = self._get_vnf_instance(instance_uuid, dst_id)
+
+            if src_vnfi is not None and dst_vnfi is not None:
+                setChaining = True
+                # re-configure the VNFs IP assignment and ensure that a new
+                # subnet is used for each E-Link
+                eline_net = ELINE_SUBNETS.pop(0)
+                ip1 = "{0}/{1}".format(str(eline_net[1]),
+                                       eline_net.prefixlen)
+                ip2 = "{0}/{1}".format(str(eline_net[2]),
+                                       eline_net.prefixlen)
+                # check if VNFs have fixed IPs (address field in VNFDs)
+                if (self._get_vnfd_cp_from_vnfi(src_vnfi, src_if_name)
+                        .get("address") is None):
+                    self._vnf_reconfigure_network(src_vnfi, src_if_name, ip1)
+                # check if VNFs have fixed IPs (address field in VNFDs)
+                if (self._get_vnfd_cp_from_vnfi(dst_vnfi, dst_if_name)
+                        .get("address") is None):
+                    self._vnf_reconfigure_network(dst_vnfi, dst_if_name, ip2)
+            # set the chaining
             if setChaining:
                 GK.net.setChain(
                     src_id, dst_id,
@@ -760,7 +592,7 @@ class Service(object):
 
             # generate lan ip address for all interfaces (of all involved (V/CDUs))
             for intf in link["connection_points_reference"]:
-                vnf_id, intf_name, _ = parse_interface(intf)
+                vnf_id, intf_name = parse_interface(intf)
                 if vnf_id is None:
                     continue  # skip references to NS connection points
                 units = self._get_vnf_instance_units(instance_uuid, vnf_id)
@@ -896,14 +728,11 @@ class Service(object):
         assert(len(GK.dcs) > 0)
         # instantiate algorithm an place
         p = algorithm()
-        p.place(self.nsd, self.vnfds, self.saps, GK.dcs)
+        p.place(self.nsd, self.vnfds, GK.dcs)
         LOG.info("Using placement algorithm: %r" % p.__class__.__name__)
         # lets print the placement result
         for name, vnfd in self.vnfds.iteritems():
             LOG.info("Placed VNF %r on DC %r" % (name, str(vnfd.get("dc"))))
-        for sap in self.saps:
-            sap_dict = self.saps[sap]
-            LOG.info("Placed SAP %r on DC %r" % (sap, str(sap_dict.get("dc"))))
 
     def _calculate_cpu_cfs_values(self, cpu_time_percentage):
         """
@@ -932,23 +761,6 @@ class Service(object):
                   (cpu_period, cpu_quota))
         return int(cpu_period), int(cpu_quota)
 
-    def check_ext_saps(self, intf_list):
-        # check if the list of interfacs contains an external SAP
-        saps_ext = [self.saps[sap]['name']
-                    for sap in self.saps if self.saps[sap]["type"] == "external"]
-        for intf_name in intf_list:
-            vnf_id, vnf_interface, vnf_sap_docker_name = parse_interface(
-                intf_name)
-            if vnf_sap_docker_name in saps_ext:
-                return vnf_sap_docker_name
-
-    def check_mgmt_interface(self, intf_list):
-        SAPs_mgmt = [p.get('id') for p in self.nsd["connection_points"]
-                     if 'management' in p.get('type')]
-        for intf_name in intf_list:
-            if intf_name in SAPs_mgmt:
-                return True
-
 
 """
 Some (simple) placement algorithms
@@ -960,7 +772,7 @@ class FirstDcPlacement(object):
     Placement: Always use one and the same data center from the GK.dcs dict.
     """
 
-    def place(self, nsd, vnfds, saps, dcs):
+    def place(self, nsd, vnfds, dcs):
         for id, vnfd in vnfds.iteritems():
             vnfd["dc"] = list(dcs.itervalues())[0]
 
@@ -970,7 +782,7 @@ class RoundRobinDcPlacement(object):
     Placement: Distribute VNFs across all available DCs in a round robin fashion.
     """
 
-    def place(self, nsd, vnfds, saps, dcs):
+    def place(self, nsd, vnfds, dcs):
         c = 0
         dcs_list = list(dcs.itervalues())
         for id, vnfd in vnfds.iteritems():
@@ -1129,6 +941,25 @@ class Exit(fr.Resource):
         list(GK.dcs.values())[0].net.stop()
 
 
+def generate_subnets(prefix, base, subnet_size=50, mask=24):
+    # Generate a list of ipaddress in subnets
+    r = list()
+    for net in range(base, base + subnet_size):
+        subnet = "{0}.{1}.0/{2}".format(prefix, net, mask)
+        r.append(ipaddress.ip_network(unicode(subnet)))
+    return r
+
+
+def reset_subnets():
+    global ELINE_SUBNETS
+    global ELAN_SUBNETS
+    # private subnet definitions for the generated interfaces
+    # 30.0.xxx.0/24
+    ELAN_SUBNETS = generate_subnets('30.0', 0, subnet_size=50, mask=24)
+    # 20.0.xxx.0/24
+    ELINE_SUBNETS = generate_subnets('20.0', 0, subnet_size=50, mask=24)
+
+
 def initialize_GK():
     global GK
     GK = Gatekeeper()
@@ -1203,20 +1034,7 @@ def parse_interface(interface_name):
     else:
         vnf_id = None
         vnf_interface = interface_name
-    return vnf_id, vnf_interface, None
-
-
-def reset_subnets():
-    # private subnet definitions for the generated interfaces
-    # 10.10.xxx.0/24
-    global SAP_SUBNETS
-    SAP_SUBNETS = generate_subnets('10.10', 0, subnet_size=50, mask=30)
-    # 10.20.xxx.0/30
-    global ELAN_SUBNETS
-    ELAN_SUBNETS = generate_subnets('10.20', 0, subnet_size=50, mask=24)
-    # 10.30.xxx.0/30
-    global ELINE_SUBNETS
-    ELINE_SUBNETS = generate_subnets('10.30', 0, subnet_size=50, mask=30)
+    return vnf_id, vnf_interface
 
 
 def get_container_name(vnf_id, vdu_id):
index af8dbf4..da5c6c3 100644 (file)
@@ -62,7 +62,6 @@ class testTangoLLCM(SimpleTestTopology):
         self.startNet()
         time.sleep(3)
 
-        print "starting tests"
         # board package
         files = {"package": open(PACKAGE_PATH, "rb")}
         r = requests.post("http://127.0.0.1:56000/packages", files=files)
@@ -97,7 +96,7 @@ class testTangoLLCM(SimpleTestTopology):
 
         # check E-Line connection, by checking the IP addresses
         for link in self.net.deployed_elines:
-            vnf_src, intf_src, vnf_sap_docker_name = parse_interface(
+            vnf_src, intf_src = parse_interface(
                 link['connection_points_reference'][0])
             print vnf_src, intf_src
             src = self.net.getNodeByName(vnf_src)
@@ -109,7 +108,7 @@ class testTangoLLCM(SimpleTestTopology):
             src_mask = [intf['netmask']
                         for intf in network_list if intf['intf_name'] == intf_src][0]
 
-            vnf_dst, intf_dst, vnf_sap_docker_name = parse_interface(
+            vnf_dst, intf_dst = parse_interface(
                 link['connection_points_reference'][1])
             dst = self.net.getNodeByName(vnf_dst)
             if not dst: