2 Playground for resource models created by University of Paderborn.
7 from emuvim
.dcemulator
.resourcemodel
import BaseResourceModel
9 LOG
= logging
.getLogger("rm.upb.simple")
10 LOG
.setLevel(logging
.DEBUG
)
15 class UpbSimpleCloudDcRM(BaseResourceModel
):
17 This will be an example resource model that limits the overall
18 resources that can be deployed per data center.
19 No over provisioning. Resources are fixed throughout entire container
23 def __init__(self
, max_cu
=32, max_mu
=1024,
24 deactivate_cpu_limit
=False,
25 deactivate_mem_limit
=False):
28 :param max_cu: Maximum number of compute units available in this DC.
29 :param max_mu: Maximum memory of entire dc.
32 self
.dc_max_cu
= max_cu
33 self
.dc_max_mu
= max_mu
36 self
.deactivate_cpu_limit
= deactivate_cpu_limit
37 self
.deactivate_mem_limit
= deactivate_mem_limit
40 super(self
.__class
__, self
).__init
__()
42 def allocate(self
, d
):
44 Allocate resources for the given container.
45 Defined by d.flavor_name
49 self
._allocated
_compute
_instances
[d
.name
] = d
50 if not self
.deactivate_cpu_limit
:
52 if not self
.deactivate_mem_limit
:
56 def _allocate_cpu(self
, d
):
58 Actually allocate (bookkeeping)
62 fl_cu
= self
._get
_flavor
(d
).get("compute")
63 # check for over provisioning
64 if self
.dc_alloc_cu
+ fl_cu
> self
.dc_max_cu
:
65 raise Exception("Not enough compute resources left.")
66 self
.dc_alloc_cu
+= fl_cu
68 def _allocate_mem(self
, d
):
70 Actually allocate (bookkeeping)
74 fl_mu
= self
._get
_flavor
(d
).get("memory")
75 # check for over provisioning
76 if self
.dc_alloc_mu
+ fl_mu
> self
.dc_max_mu
:
77 raise Exception("Not enough memory resources left.")
78 self
.dc_alloc_mu
+= fl_mu
82 Free resources allocated to the given container.
86 del self
._allocated
_compute
_instances
[d
.name
]
87 if not self
.deactivate_cpu_limit
:
89 if not self
.deactivate_mem_limit
:
93 def _free_cpu(self
, d
):
99 self
.dc_alloc_cu
-= self
._get
_flavor
(d
).get("compute")
101 def _free_mem(self
, d
):
107 self
.dc_alloc_mu
-= self
._get
_flavor
(d
).get("memory")
109 def _apply_limits(self
):
111 Recalculate real resource limits for all allocated containers and apply them
113 We have to recalculate for all to allow e.g. overprovisioning models.
116 for d
in self
._allocated
_compute
_instances
.itervalues():
117 if not self
.deactivate_cpu_limit
:
118 self
._apply
_cpu
_limits
(d
)
119 if not self
.deactivate_mem_limit
:
120 self
._apply
_mem
_limits
(d
)
122 def _apply_cpu_limits(self
, d
):
124 Calculate real CPU limit (CFS bandwidth) and apply.
128 number_cu
= self
._get
_flavor
(d
).get("compute")
129 # get cpu time fraction for entire emulation
130 e_cpu
= self
.registrar
.e_cpu
131 # calculate cpu time fraction of a single compute unit
132 self
.single_cu
= float(e_cpu
) / sum([rm
.dc_max_cu
for rm
in list(self
.registrar
.resource_models
)])
133 # calculate cpu time fraction for container with given flavor
134 cpu_time_percentage
= self
.single_cu
* number_cu
135 # calculate input values for CFS scheduler bandwidth limitation
136 cpu_period
, cpu_quota
= self
._calculate
_cpu
_cfs
_values
(cpu_time_percentage
)
137 # apply limits to container if changed
138 if d
.cpu_period
!= cpu_period
or d
.cpu_quota
!= cpu_quota
:
139 LOG
.debug("Setting CPU limit for %r: cpu_quota = cpu_period * limit = %f * %f = %f" % (
140 d
.name
, cpu_period
, cpu_time_percentage
, cpu_quota
))
141 d
.updateCpuLimit(cpu_period
=int(cpu_period
), cpu_quota
=int(cpu_quota
))
143 def _calculate_cpu_cfs_values(self
, cpu_time_percentage
):
145 Calculate cpu period and quota for CFS
146 :param cpu_time_percentage: percentage of overall CPU to be used
147 :return: cpu_period, cpu_quota
149 # (see: https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt)
150 # Attention minimum cpu_quota is 1ms (micro)
151 cpu_period
= CPU_PERIOD
# lets consider a fixed period of 1000000 microseconds for now
152 cpu_quota
= cpu_period
* cpu_time_percentage
# calculate the fraction of cpu time for this container
153 # ATTENTION >= 1000 to avoid a invalid argument system error ... no idea why
156 LOG
.warning("Increased CPU quota to avoid system error.")
157 return cpu_period
, cpu_quota
159 def _apply_mem_limits(self
, d
):
161 Calculate real mem limit and apply.
165 number_mu
= self
._get
_flavor
(d
).get("memory")
166 # get memory amount for entire emulation
167 e_mem
= self
.registrar
.e_mem
168 # calculate amount of memory for a single mu
169 self
.single_mu
= float(e_mem
) / sum([rm
.dc_max_mu
for rm
in list(self
.registrar
.resource_models
)])
170 # calculate mem for given flavor
171 mem_limit
= self
.single_mu
* number_mu
172 mem_limit
= self
._calculate
_mem
_limit
_value
(mem_limit
)
173 # apply to container if changed
174 if d
.mem_limit
!= mem_limit
:
175 LOG
.debug("Setting MEM limit for %r: mem_limit = %f MB" % (d
.name
, mem_limit
/1024/1024))
176 d
.updateMemoryLimit(mem_limit
=mem_limit
)
178 def _calculate_mem_limit_value(self
, mem_limit
):
180 Calculate actual mem limit as input for cgroup API
181 :param mem_limit: abstract mem limit
182 :return: concrete mem limit
184 # ATTENTION minimum mem_limit per container is 4MB
187 LOG
.warning("Increased MEM limit because it was less than 4.0 MB.")
189 return int(mem_limit
*1024*1024)
191 def get_state_dict(self
):
193 Return the state of the resource model as simple dict.
194 Helper method for logging functionality.
197 # collect info about all allocated instances
198 allocation_state
= dict()
199 for k
, d
in self
._allocated
_compute
_instances
.iteritems():
201 s
["cpu_period"] = d
.cpu_period
202 s
["cpu_quota"] = d
.cpu_quota
203 s
["cpu_shares"] = d
.cpu_shares
204 s
["mem_limit"] = d
.mem_limit
205 s
["memswap_limit"] = d
.memswap_limit
206 allocation_state
[k
] = s
209 r
["e_cpu"] = self
.registrar
.e_cpu
210 r
["e_mem"] = self
.registrar
.e_mem
211 r
["dc_max_cu"] = self
.dc_max_cu
212 r
["dc_max_mu"] = self
.dc_max_mu
213 r
["dc_alloc_cu"] = self
.dc_alloc_cu
214 r
["dc_alloc_mu"] = self
.dc_alloc_mu
215 r
["single_cu_percentage"] = self
.single_cu
216 r
["single_mu_percentage"] = self
.single_mu
217 r
["allocation_state"] = allocation_state
220 def _get_flavor(self
, d
):
222 Get flavor assigned to given container.
223 Identified by d.flavor_name.
227 if d
.flavor_name
not in self
._flavors
:
228 raise Exception("Flavor %r does not exist" % d
.flavor_name
)
229 return self
._flavors
.get(d
.flavor_name
)
231 def _write_log(self
, d
, path
, action
):
233 Helper to log RM info for experiments.
235 :param path: log path
236 :param action: allocate or free
241 # we have a path: write out RM info
244 l
["container_state"] = d
.getStatus()
246 l
["rm_state"] = self
.get_state_dict()
248 with
open(path
, "a") as f
:
249 f
.write("%s\n" % json
.dumps(l
))