425d959d1e31e60445ef34c93003b760fda3a1e9
2 Playground for resource models created by University of Paderborn.
7 from emuvim
.dcemulator
.resourcemodel
import BaseResourceModel
9 LOG
= logging
.getLogger("rm.upb.simple")
10 LOG
.setLevel(logging
.DEBUG
)
15 class UpbSimpleCloudDcRM(BaseResourceModel
):
17 This will be an example resource model that limits the overall
18 resources that can be deployed per data center.
19 No over provisioning. Resources are fixed throughout entire container
23 def __init__(self
, max_cu
=32, max_mu
=1024,
24 deactivate_cpu_limit
=False,
25 deactivate_mem_limit
=False):
28 :param max_cu: Maximum number of compute units available in this DC.
29 :param max_mu: Maximum memory of entire dc.
32 self
.dc_max_cu
= max_cu
33 self
.dc_max_mu
= max_mu
36 self
.deactivate_cpu_limit
= deactivate_cpu_limit
37 self
.deactivate_mem_limit
= deactivate_mem_limit
38 super(self
.__class
__, self
).__init
__()
40 def allocate(self
, d
):
42 Allocate resources for the given container.
43 Defined by d.flavor_name
47 self
._allocated
_compute
_instances
[d
.name
] = d
48 if not self
.deactivate_cpu_limit
:
50 if not self
.deactivate_mem_limit
:
54 def _allocate_cpu(self
, d
):
56 Actually allocate (bookkeeping)
60 fl_cu
= self
._get
_flavor
(d
).get("compute")
61 # check for over provisioning
62 if self
.dc_alloc_cu
+ fl_cu
> self
.dc_max_cu
:
63 raise Exception("Not enough compute resources left.")
64 self
.dc_alloc_cu
+= fl_cu
66 def _allocate_mem(self
, d
):
68 Actually allocate (bookkeeping)
72 fl_mu
= self
._get
_flavor
(d
).get("memory")
73 # check for over provisioning
74 if self
.dc_alloc_mu
+ fl_mu
> self
.dc_max_mu
:
75 raise Exception("Not enough memory resources left.")
76 self
.dc_alloc_mu
+= fl_mu
80 Free resources allocated to the given container.
84 del self
._allocated
_compute
_instances
[d
.name
]
85 if not self
.deactivate_cpu_limit
:
87 if not self
.deactivate_mem_limit
:
91 def _free_cpu(self
, d
):
97 self
.dc_alloc_cu
-= self
._get
_flavor
(d
).get("compute")
99 def _free_mem(self
, d
):
105 self
.dc_alloc_mu
-= self
._get
_flavor
(d
).get("memory")
107 def _apply_limits(self
):
109 Recalculate real resource limits for all allocated containers and apply them
111 We have to recalculate for all to allow e.g. overprovisioning models.
114 for d
in self
._allocated
_compute
_instances
.itervalues():
115 if not self
.deactivate_cpu_limit
:
116 self
._apply
_cpu
_limits
(d
)
117 if not self
.deactivate_mem_limit
:
118 self
._apply
_mem
_limits
(d
)
120 def _apply_cpu_limits(self
, d
):
122 Calculate real CPU limit (CFS bandwidth) and apply.
126 number_cu
= self
._get
_flavor
(d
).get("compute")
127 # get cpu time fraction for entire emulation
128 e_cpu
= self
.registrar
.e_cpu
129 # calculate cpu time fraction of a single compute unit
130 single_cu
= float(e_cpu
) / sum([rm
.dc_max_cu
for rm
in list(self
.registrar
.resource_models
)])
131 # calculate cpu time fraction for container with given flavor
132 cpu_time_percentage
= self
.single_cu
* number_cu
133 # calculate input values for CFS scheduler bandwidth limitation
134 cpu_period
, cpu_quota
= self
._calculate
_cpu
_cfs
_values
(cpu_time_percentage
)
135 # apply limits to container if changed
136 if d
.cpu_period
!= cpu_period
or d
.cpu_quota
!= cpu_quota
:
137 LOG
.debug("Setting CPU limit for %r: cpu_quota = cpu_period * limit = %f * %f = %f" % (
138 d
.name
, cpu_period
, cpu_time_percentage
, cpu_quota
))
139 d
.updateCpuLimit(cpu_period
=int(cpu_period
), cpu_quota
=int(cpu_quota
))
141 def _calculate_cpu_cfs_values(self
, cpu_time_percentage
):
143 Calculate cpu period and quota for CFS
144 :param cpu_time_percentage: percentage of overall CPU to be used
145 :return: cpu_period, cpu_quota
147 # (see: https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt)
148 # Attention minimum cpu_quota is 1ms (micro)
149 cpu_period
= CPU_PERIOD
# lets consider a fixed period of 1000000 microseconds for now
150 cpu_quota
= cpu_period
* cpu_time_percentage
# calculate the fraction of cpu time for this container
151 # ATTENTION >= 1000 to avoid a invalid argument system error ... no idea why
154 LOG
.warning("Increased CPU quota to avoid system error.")
155 return cpu_period
, cpu_quota
157 def _apply_mem_limits(self
, d
):
159 Calculate real mem limit and apply.
163 number_mu
= self
._get
_flavor
(d
).get("memory")
164 # get memory amount for entire emulation
165 e_mem
= self
.registrar
.e_mem
166 # calculate amount of memory for a single mu
167 single_mu
= float(e_mem
) / sum([rm
.dc_max_mu
for rm
in list(self
.registrar
.resource_models
)])
168 # calculate mem for given flavor
169 mem_limit
= self
.single_mu
* number_mu
170 mem_limit
= self
._calculate
_mem
_limit
_value
(mem_limit
)
171 # apply to container if changed
172 if d
.mem_limit
!= mem_limit
:
173 LOG
.debug("Setting MEM limit for %r: mem_limit = %f MB" % (d
.name
, mem_limit
/1024/1024))
174 d
.updateMemoryLimit(mem_limit
=mem_limit
)
176 def _calculate_mem_limit_value(self
, mem_limit
):
178 Calculate actual mem limit as input for cgroup API
179 :param mem_limit: abstract mem limit
180 :return: concrete mem limit
182 # ATTENTION minimum mem_limit per container is 4MB
185 LOG
.warning("Increased MEM limit because it was less than 4.0 MB.")
187 return int(mem_limit
*1024*1024)
189 def get_state_dict(self
):
191 Return the state of the resource model as simple dict.
192 Helper method for logging functionality.
195 # collect info about all allocated instances
196 allocation_state
= dict()
197 for k
, d
in self
._allocated
_compute
_instances
.iteritems():
199 s
["cpu_period"] = d
.cpu_period
200 s
["cpu_quota"] = d
.cpu_quota
201 s
["cpu_shares"] = d
.cpu_shares
202 s
["mem_limit"] = d
.mem_limit
203 s
["memswap_limit"] = d
.memswap_limit
204 allocation_state
[k
] = s
207 r
["e_cpu"] = self
.registrar
.e_cpu
208 r
["e_mem"] = self
.registrar
.e_mem
209 r
["dc_max_cu"] = self
.dc_max_cu
210 r
["dc_max_mu"] = self
.dc_max_mu
211 r
["dc_alloc_cu"] = self
.dc_alloc_cu
212 r
["dc_alloc_mu"] = self
.dc_alloc_mu
213 r
["allocation_state"] = allocation_state
216 def _get_flavor(self
, d
):
218 Get flavor assigned to given container.
219 Identified by d.flavor_name.
223 if d
.flavor_name
not in self
._flavors
:
224 raise Exception("Flavor %r does not exist" % d
.flavor_name
)
225 return self
._flavors
.get(d
.flavor_name
)
227 def _write_log(self
, d
, path
, action
):
229 Helper to log RM info for experiments.
231 :param path: log path
232 :param action: allocate or free
237 # we have a path: write out RM info
240 l
["container_state"] = d
.getStatus()
242 l
["rm_state"] = self
.get_state_dict()
244 with
open(path
, "a") as f
:
245 f
.write("%s\n" % json
.dumps(l
))