From 60bf8b8b3bf5e800bdb761f2f403ef2944262137 Mon Sep 17 00:00:00 2001 From: peusterm Date: Wed, 6 Apr 2016 14:12:35 +0200 Subject: [PATCH] Added logging functionality to resource models. --- src/emuvim/dcemulator/net.py | 4 +- src/emuvim/dcemulator/node.py | 38 ++++++++++++++++++- .../dcemulator/resourcemodel/__init__.py | 18 ++++++--- .../dcemulator/resourcemodel/upb/simple.py | 16 ++++++++ .../examples/resource_model_demo_topology.py | 26 ++++++++++--- src/emuvim/test/test_resourcemodel_api.py | 14 +++---- 6 files changed, 96 insertions(+), 20 deletions(-) diff --git a/src/emuvim/dcemulator/net.py b/src/emuvim/dcemulator/net.py index 80fb2f8..ff53773 100755 --- a/src/emuvim/dcemulator/net.py +++ b/src/emuvim/dcemulator/net.py @@ -62,13 +62,13 @@ class DCNetwork(Dockernet): self.rm_registrar = ResourceModelRegistrar( dc_emulation_max_cpu, dc_emulation_max_mem) - def addDatacenter(self, label, metadata={}): + def addDatacenter(self, label, metadata={}, resource_log_path=None): """ Create and add a logical cloud data center to the network. """ if label in self.dcs: raise Exception("Data center label already exists: %s" % label) - dc = Datacenter(label, metadata=metadata) + dc = Datacenter(label, metadata=metadata, resource_log_path=resource_log_path) dc.net = self # set reference to network self.dcs[label] = dc dc.create() # finally create the data center in our Mininet instance diff --git a/src/emuvim/dcemulator/node.py b/src/emuvim/dcemulator/node.py index 7a7fa78..3f76df9 100755 --- a/src/emuvim/dcemulator/node.py +++ b/src/emuvim/dcemulator/node.py @@ -5,6 +5,8 @@ Distributed Cloud Emulator (dcemulator) from mininet.node import Docker from mininet.link import Link import logging +import time +import json LOG = logging.getLogger("dcemulator") LOG.setLevel(logging.DEBUG) @@ -70,7 +72,7 @@ class Datacenter(object): DC_COUNTER = 1 - def __init__(self, label, metadata={}): + def __init__(self, label, metadata={}, resource_log_path=None): self.net = None # DCNetwork to which we belong # each node (DC) has a short internal name used by Mininet # this is caused by Mininets naming limitations for swtiches etc. @@ -80,6 +82,8 @@ class Datacenter(object): self.label = label # dict to store arbitrary metadata (e.g. latitude and longitude) self.metadata = metadata + # path to which resource information should be logged (e.g. for experiments). None = no logging + self.resource_log_path = resource_log_path # first prototype assumes one "bigswitch" per DC self.switch = None # keep track of running containers @@ -183,6 +187,22 @@ class Datacenter(object): self.net.addLink(d, self.switch, params1=nw, cls=Link) # do bookkeeping self.containers[name] = d + + # write resource log if a path is given + if self.resource_log_path is not None: + l = dict() + l["t"] = time.time() + l["name"] = name + l["compute"] = d.getStatus() + l["flavor_name"] = flavor_name + l["action"] = "allocate" + l["cpu_limit"] = cpu_limit + l["mem_limit"] = mem_limit + l["disk_limit"] = disk_limit + l["rm_state"] = None if self._resource_model is None else self._resource_model.get_state_dict() + # append to logfile + with open(self.resource_log_path, "a") as f: + f.write("%s\n" % json.dumps(l)) return d # we might use UUIDs for naming later on def stopCompute(self, name): @@ -192,6 +212,7 @@ class Datacenter(object): assert name is not None if name not in self.containers: raise Exception("Container with name %s not found." % name) + LOG.debug("Stopping compute instance %r in data center %r" % (name, str(self))) self.net.removeLink( link=None, node1=self.containers[name], node2=self.switch) self.net.removeDocker("%s" % (name)) @@ -199,6 +220,21 @@ class Datacenter(object): # call resource model and free resources if self._resource_model is not None: self._resource_model.free(name) + + # write resource log if a path is given + if self.resource_log_path is not None: + l = dict() + l["t"] = time.time() + l["name"] = name + l["flavor_name"] = None + l["action"] = "free" + l["cpu_limit"] = -1 + l["mem_limit"] = -1 + l["disk_limit"] = -1 + l["rm_state"] = None if self._resource_model is None else self._resource_model.get_state_dict() + # append to logfile + with open(self.resource_log_path, "a") as f: + f.write("%s\n" % json.dumps(l)) return True def listCompute(self): diff --git a/src/emuvim/dcemulator/resourcemodel/__init__.py b/src/emuvim/dcemulator/resourcemodel/__init__.py index e35da25..c28e226 100644 --- a/src/emuvim/dcemulator/resourcemodel/__init__.py +++ b/src/emuvim/dcemulator/resourcemodel/__init__.py @@ -87,15 +87,15 @@ class BaseResourceModel(object): initialize some default flavours (naming/sizes inspired by OpenStack) """ self.addFlavour(ResourceFlavor( - "tiny", {"compute": 1, "memory": 32, "disk": 1})) + "tiny", {"compute": 0.5, "memory": 32, "disk": 1})) self.addFlavour(ResourceFlavor( - "small", {"compute": 4, "memory": 128, "disk": 20})) + "small", {"compute": 1.0, "memory": 128, "disk": 20})) self.addFlavour(ResourceFlavor( - "medium", {"compute": 8, "memory": 256, "disk": 40})) + "medium", {"compute": 4.0, "memory": 256, "disk": 40})) self.addFlavour(ResourceFlavor( - "large", {"compute": 16, "memory": 512, "disk": 80})) + "large", {"compute": 8.0, "memory": 512, "disk": 80})) self.addFlavour(ResourceFlavor( - "xlarge", {"compute": 32, "memory": 1024, "disk": 160})) + "xlarge", {"compute": 16.0, "memory": 1024, "disk": 160})) def addFlavour(self, fl): """ @@ -127,3 +127,11 @@ class BaseResourceModel(object): LOG.warning("Free in BaseResourceModel: %r" % name) del self.allocated_compute_instances[name] return True + + def get_state_dict(self): + """ + Return the state of the resource model as simple dict. + Helper method for logging functionality. + :return: + """ + return dict() diff --git a/src/emuvim/dcemulator/resourcemodel/upb/simple.py b/src/emuvim/dcemulator/resourcemodel/upb/simple.py index 84aa7a9..8e96dba 100644 --- a/src/emuvim/dcemulator/resourcemodel/upb/simple.py +++ b/src/emuvim/dcemulator/resourcemodel/upb/simple.py @@ -59,6 +59,22 @@ class UpbSimpleCloudDcRM(BaseResourceModel): # we don't have to calculate anything special here in this simple model return True + def get_state_dict(self): + """ + Return the state of the resource model as simple dict. + Helper method for logging functionality. + :return: + """ + r = dict() + r["e_cpu"] = self.registrar.e_cpu + r["e_mem"] = self.registrar.e_mem + r["dc_max_cu"] = self.dc_max_cu + r["dc_max_mu"] = self.dc_max_mu + r["dc_alloc_cu"] = self.dc_alloc_cu + r["dc_alloc_mu"] = self.dc_alloc_mu + r["allocated_compute_instances"] = self.allocated_compute_instances + return r + def _allocate_cpu(self, flavor): """ Allocate CPU time. diff --git a/src/emuvim/examples/resource_model_demo_topology.py b/src/emuvim/examples/resource_model_demo_topology.py index ae6ba5e..7a39b00 100644 --- a/src/emuvim/examples/resource_model_demo_topology.py +++ b/src/emuvim/examples/resource_model_demo_topology.py @@ -4,6 +4,7 @@ A simple topology to test resource model support. import logging import time +import os from mininet.log import setLogLevel from mininet.node import Controller from emuvim.dcemulator.net import DCNetwork @@ -14,20 +15,24 @@ from emuvim.dcemulator.resourcemodel.upb.simple import UpbSimpleCloudDcRM logging.basicConfig(level=logging.INFO) +RESOURCE_LOG_PATH = "resource.log" + + def create_topology1(): + cleanup() # create topology # use a maximum of 50% cpu time for containers added to data centers net = DCNetwork(dc_emulation_max_cpu=0.5, controller=Controller) # add some data centers and create a topology - dc1 = net.addDatacenter("dc1") - dc2 = net.addDatacenter("dc2") + dc1 = net.addDatacenter("dc1", resource_log_path=RESOURCE_LOG_PATH) + dc2 = net.addDatacenter("dc2", resource_log_path=RESOURCE_LOG_PATH) s1 = net.addSwitch("s1") net.addLink(dc1, s1, delay="10ms") net.addLink(dc2, s1, delay="20ms") # create and assign resource models for each DC - rm1 = UpbSimpleCloudDcRM(max_cu=10, max_mu=1024) - rm2 = UpbSimpleCloudDcRM(max_cu=20) + rm1 = UpbSimpleCloudDcRM(max_cu=4, max_mu=1024) + rm2 = UpbSimpleCloudDcRM(max_cu=6) dc1.assignResourceModel(rm1) dc2.assignResourceModel(rm2) @@ -54,11 +59,22 @@ def create_topology1(): dc1.startCompute("vnf3", flavor_name="small") dc2.startCompute("vnf4", flavor_name="medium") print "... done." - time.sleep(2) + time.sleep(5) + print "Removing instances ..." + dc1.stopCompute("vnf1") + dc2.stopCompute("vnf4") + print "... done" net.CLI() net.stop() +def cleanup(): + try: + os.remove(RESOURCE_LOG_PATH) + except OSError: + pass + + def main(): setLogLevel('info') # set Mininet loglevel create_topology1() diff --git a/src/emuvim/test/test_resourcemodel_api.py b/src/emuvim/test/test_resourcemodel_api.py index 0836c68..5266330 100644 --- a/src/emuvim/test/test_resourcemodel_api.py +++ b/src/emuvim/test/test_resourcemodel_api.py @@ -85,27 +85,27 @@ class testUpbSimpleCloudDcRM(SimpleTestTopology): reg.register("test_dc", rm) res = rm.allocate("c1", "tiny") # calculate allocation - self.assertEqual(res[0], E_CPU / MAX_CU * 1) # validate compute result + self.assertEqual(res[0], E_CPU / MAX_CU * 0.5) # validate compute result self.assertEqual(res[1], float(E_MEM) / MAX_MU * 32) # validate memory result self.assertTrue(res[2] < 0) # validate disk result res = rm.allocate("c2", "small") # calculate allocation - self.assertEqual(res[0], E_CPU / MAX_CU * 4) # validate compute result + self.assertEqual(res[0], E_CPU / MAX_CU * 1) # validate compute result self.assertEqual(res[1], float(E_MEM) / MAX_MU * 128) # validate memory result self.assertTrue(res[2] < 0) # validate disk result res = rm.allocate("c3", "medium") # calculate allocation - self.assertEqual(res[0], E_CPU / MAX_CU * 8) # validate compute result + self.assertEqual(res[0], E_CPU / MAX_CU * 4) # validate compute result self.assertEqual(res[1], float(E_MEM) / MAX_MU * 256) # validate memory result self.assertTrue(res[2] < 0) # validate disk result res = rm.allocate("c4", "large") # calculate allocation - self.assertEqual(res[0], E_CPU / MAX_CU * 16) # validate compute result + self.assertEqual(res[0], E_CPU / MAX_CU * 8) # validate compute result self.assertEqual(res[1], float(E_MEM) / MAX_MU * 512) # validate memory result self.assertTrue(res[2] < 0) # validate disk result res = rm.allocate("c5", "xlarge") # calculate allocation - self.assertEqual(res[0], E_CPU / MAX_CU * 32) # validate compute result + self.assertEqual(res[0], E_CPU / MAX_CU * 16) # validate compute result self.assertEqual(res[1], float(E_MEM) / MAX_MU * 1024) # validate memory result self.assertTrue(res[2] < 0) # validate disk result @@ -116,7 +116,7 @@ class testUpbSimpleCloudDcRM(SimpleTestTopology): """ # config E_CPU = 1.0 - MAX_CU = 100 + MAX_CU = 40 E_MEM = 512 MAX_MU = 4096 # create dummy resource model environment @@ -175,7 +175,7 @@ class testUpbSimpleCloudDcRM(SimpleTestTopology): rm = UpbSimpleCloudDcRM(max_cu=100, max_mu=100) reg.register("test_dc", rm) rm.allocate("c1", "tiny") # calculate allocation - self.assertTrue(rm.dc_alloc_cu == 1) + self.assertTrue(rm.dc_alloc_cu == 0.5) rm.free("c1") self.assertTrue(rm.dc_alloc_cu == 0) -- 2.25.1