1 # -*- coding: utf-8 -*-
5 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
6 # This file is part of openmano
9 # Licensed under the Apache License, Version 2.0 (the "License"); you may
10 # not use this file except in compliance with the License. You may obtain
11 # a copy of the License at
13 # http://www.apache.org/licenses/LICENSE-2.0
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
17 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
18 # License for the specific language governing permissions and limitations
21 # For those usages not covered by the Apache License, Version 2.0 please
22 # contact with: nfvlabs@tid.es
26 Implement the logic for obtaining compute nodes information
27 Resource Availability Descriptor
29 __author__
="Pablo Montes"
31 #TODO: remove warnings, remove unused things
33 from definitionsClass
import definitionsClass
34 from auxiliary_functions
import get_ssh_connection
36 from xml
.etree
import ElementTree
42 def getCredentials(creds
, data
):
43 """Used as a backup for libvirt.openAuth in order to provide password that came with data,
44 not used by the moment
46 print "RADclass:getCredentials", creds
, data
49 if cred
[0] == libvirt
.VIR_CRED_AUTHNAME
:
51 elif cred
[0] == libvirt
.VIR_CRED_PASSPHRASE
:
63 self
.nodes
= dict() #Dictionary of nodes. Keys are the node id, values are Node() elements
64 self
.nr_processors
= None #Integer. Number of processors in the system
65 self
.processor_family
= None #If all nodes have the same value equal them, otherwise keep as None
66 self
.processor_manufacturer
= None #If all nodes have the same value equal them, otherwise keep as None
67 self
.processor_version
= None #If all nodes have the same value equal them, otherwise keep as None
68 self
.processor_features
= None #If all nodes have the same value equal them, otherwise keep as None
69 self
.memory_type
= None #If all nodes have the same value equal them, otherwise keep as None
70 self
.memory_freq
= None #If all nodes have the same value equal them, otherwise keep as None
71 self
.memory_nr_channels
= None #If all nodes have the same value equal them, otherwise keep as None
72 self
.memory_size
= None #Integer. Sum of the memory in all nodes
73 self
.memory_hugepage_sz
= None
74 self
.hypervisor
= Hypervisor() #Hypervisor information
75 self
.os
= OpSys() #Operating system information
76 self
.ports_list
= list() #List containing all network ports in the node. This is used to avoid having defined multiple times the same port in the system
79 def obtain_RAD(self
, user
, password
, machine
):
80 """This function obtains the RAD information from the remote server.
81 It uses both a ssh and a libvirt connection.
82 It is desirable in future versions get rid of the ssh connection, but currently
83 libvirt does not provide all the needed information.
84 Returns (True, Warning) in case of success and (False, <error>) in case of error"""
87 #Get virsh and ssh connection
88 (return_status
, code
) = get_ssh_connection(machine
, user
, password
)
90 print 'RADclass.obtain_RAD() error:', code
91 return (return_status
, code
)
94 self
.connection_IP
= machine
95 #print "libvirt open pre"
96 virsh_conn
=libvirt
.open("qemu+ssh://"+user
+'@'+machine
+"/system")
97 #virsh_conn=libvirt.openAuth("qemu+ssh://"+user+'@'+machine+"/system",
98 # [[libvirt.VIR_CRED_AUTHNAME, libvirt.VIR_CRED_PASSPHRASE, libvirt.VIR_CRED_USERNAME], getCredentials, password],
100 #print "libvirt open after"
102 # #Set connection infomation
103 # (return_status, code) = self.set_connection_info(machine, user, password)
104 # if not return_status:
105 # return (return_status, 'Error in '+machine+': '+code)
108 machine_name
= get_hostname(virsh_conn
)
109 (return_status
, code
) = self
.set_name(machine_name
)
110 if not return_status
:
111 return (return_status
, 'Error at self.set_name in '+machine
+': '+code
)
114 #Get the server processors information
116 (return_status
, code
) = get_processor_information(ssh_conn
, virsh_conn
, processors
)
117 if not return_status
:
118 return (return_status
, 'Error at get_processor_information in '+machine
+': '+code
)
121 #Get the server memory information
122 memory_nodes
= dict()
123 (return_status
, code
) = get_memory_information(ssh_conn
, virsh_conn
, memory_nodes
)
124 if not return_status
:
125 return (return_status
, 'Error at get_memory_information in '+machine
+': '+code
)
128 #Get nics information
129 nic_topology
= dict()
130 # (return_status, code) = get_nic_information_old(ssh_conn, nic_topology)
131 (return_status
, code
) = get_nic_information(ssh_conn
, virsh_conn
, nic_topology
)
132 if not return_status
:
133 return (return_status
, 'Error at get_nic_information in '+machine
+': '+code
)
136 #Pack each processor, memory node and nics in a node element
137 #and add the node to the RAD element
138 for socket_id
, processor
in processors
.iteritems():
140 if not socket_id
in nic_topology
:
141 nic_topology
[socket_id
] = list()
143 (return_status
, code
) = node
.set(processor
, memory_nodes
[socket_id
], nic_topology
[socket_id
])
145 # (return_status, code) = node.set(processor, memory_nodes[socket_id])
146 if not return_status
:
147 return (return_status
, 'Error at node.set in '+machine
+': '+code
)
149 (return_status
, code
) = self
.insert_node(node
)
150 if not return_status
:
151 return (return_status
, 'Error at self.insert_node in '+machine
+': '+code
)
152 if code
not in warning_text
:
157 (return_status
, code
) = get_os_information(ssh_conn
, os
)
158 if not return_status
:
159 return (return_status
, 'Error at get_os_information in '+machine
+': '+code
)
161 (return_status
, code
) = self
.set_os(os
)
162 if not return_status
:
163 return (return_status
, 'Error at self.set_os in '+machine
+': '+code
)
166 #Fill hypervisor data
167 hypervisor
= Hypervisor()
168 (return_status
, code
) = get_hypervisor_information(virsh_conn
, hypervisor
)
169 if not return_status
:
170 return (return_status
, 'Error at get_hypervisor_information in '+machine
+': '+code
)
172 (return_status
, code
) = self
.set_hypervisor(hypervisor
)
173 if not return_status
:
174 return (return_status
, 'Error at self.set_hypervisor in '+machine
+': '+code
)
178 return (True, warning_text
)
179 except libvirt
.libvirtError
, e
:
180 text
= e
.get_error_message()
181 print 'RADclass.obtain_RAD() exception:', text
183 except paramiko
.ssh_exception
.SSHException
, e
:
185 print "obtain_RAD ssh Exception:", text
188 def set_name(self
,name
):
189 """Sets the machine name.
190 Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
191 if not isinstance(name
,str):
192 return (False, 'The variable \'name\' must be text')
196 def set_connection_info(self
, machine
, user
, password
):
197 """Sets the connection information.
198 Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
199 if not isinstance(machine
,str):
200 return (False, 'The variable \'machine\' must be text')
201 if not isinstance(user
,str):
202 return (False, 'The variable \'user\' must be text')
203 # if not isinstance(password,str):
204 # return (False, 'The variable \'password\' must be text')
205 (self
.machine
, self
.user
, self
.password
) = (machine
, user
, password
)
208 def insert_node(self
,node
):
209 """Inserts a new node and updates class variables.
210 Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
211 if not isinstance(node
,Node
):
212 return (False, 'The variable \'node\' must be a Node element')
214 if node
.id_
in self
.nodes
:
215 return (False, 'The node is already present in the nodes list.')
217 #Check if network ports have not been inserted previously as part of another node
218 for port_key
in node
.ports_list
:
219 if port_key
in self
.ports_list
:
220 return (False, 'Network port '+port_key
+' defined multiple times in the system')
221 self
.ports_list
.append(port_key
)
224 self
.nodes
[node
.id_
] = node
227 self
.update_variables()
231 def update_variables(self
):
232 """Updates class variables.
233 Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
235 #The number of processors and nodes is the same
236 self
.nr_processors
= len(self
.nodes
)
238 #If all processors are the same get the values. Otherwise keep them as none
239 prev_processor_family
= prev_processor_manufacturer
= prev_processor_version
= prev_processor_features
= None
240 different_processor_family
= different_processor_manufacturer
= different_processor_version
= different_processor_features
= False
241 for node
in self
.nodes
.itervalues():
242 (self
.processor_family
, self
.processor_manufacturer
, self
.processor_version
, self
.processor_features
) = node
.get_processor_info()
243 if prev_processor_family
!= None and self
.processor_family
!= prev_processor_family
:
244 different_processor_family
= True
245 if prev_processor_manufacturer
!= None and self
.processor_manufacturer
!= prev_processor_manufacturer
:
246 different_processor_manufacturer
= True
247 if prev_processor_version
!= None and self
.processor_version
!= prev_processor_version
:
248 different_processor_version
= True
249 if prev_processor_features
!= None and self
.processor_features
!= prev_processor_features
:
250 different_processor_features
= True
251 (prev_processor_family
, prev_processor_manufacturer
, prev_processor_version
, prev_processor_features
) = (self
.processor_family
, self
.processor_manufacturer
, self
.processor_version
, self
.processor_features
)
253 if different_processor_family
:
254 self
.processor_family
= None
255 if different_processor_features
:
256 self
.processor_features
= None
257 if different_processor_manufacturer
:
258 self
.processor_manufacturer
= None
259 if different_processor_version
:
260 self
.processor_version
= None
262 #If all memory nodes are the same get the values. Otherwise keep them as none
263 #Sum the total memory
265 different_memory_freq
= different_memory_nr_channels
= different_memory_type
= different_memory_hugepage_sz
= False
266 prev_memory_freq
= prev_memory_nr_channels
= prev_memory_type
= prev_memory_hugepage_sz
= None
267 for node
in self
.nodes
.itervalues():
268 (self
.memory_freq
, self
.memory_nr_channels
, self
.memory_type
, memory_size
, self
.memory_hugepage_sz
) = node
.get_memory_info()
269 self
.memory_size
+= memory_size
270 if prev_memory_freq
!= None and self
.memory_freq
!= prev_memory_freq
:
271 different_memory_freq
= True
272 if prev_memory_nr_channels
!= None and self
.memory_nr_channels
!= prev_memory_nr_channels
:
273 different_memory_nr_channels
= True
274 if prev_memory_type
!= None and self
.memory_type
!= prev_memory_type
:
275 different_memory_type
= True
276 if prev_memory_hugepage_sz
!= None and self
.memory_hugepage_sz
!= prev_memory_hugepage_sz
:
277 different_memory_hugepage_sz
= True
278 (prev_memory_freq
, prev_memory_nr_channels
, prev_memory_type
, prev_memory_hugepage_sz
) = (self
.memory_freq
, self
.memory_nr_channels
, self
.memory_type
, self
.memory_hugepage_sz
)
280 if different_memory_freq
:
281 self
.memory_freq
= None
282 if different_memory_nr_channels
:
283 self
.memory_nr_channels
= None
284 if different_memory_type
:
285 self
.memory_type
= None
286 if different_memory_hugepage_sz
:
287 warning_text
+= 'Detected different hugepages size in different sockets\n'
289 return (True, warning_text
)
291 def set_hypervisor(self
,hypervisor
):
292 """Sets the hypervisor.
293 Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
294 if not isinstance(hypervisor
,Hypervisor
):
295 return (False, 'The variable \'hypervisor\' must be of class Hypervisor')
297 self
.hypervisor
.assign(hypervisor
)
301 """Sets the operating system.
302 Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
303 if not isinstance(os
,OpSys
):
304 return (False, 'The variable \'os\' must be of class OpSys')
310 text
= 'name: '+str(self
.name
)+'\n'
311 text
+= 'processor:\n'
312 text
+= ' nr_processors: '+str(self
.nr_processors
)+'\n'
313 text
+= ' family: '+str(self
.processor_family
)+'\n'
314 text
+= ' manufacturer: '+str(self
.processor_manufacturer
)+'\n'
315 text
+= ' version: '+str(self
.processor_version
)+'\n'
316 text
+= ' features: '+str(self
.processor_features
)+'\n'
318 text
+= ' type: '+str(self
.memory_type
)+'\n'
319 text
+= ' freq: '+str(self
.memory_freq
)+'\n'
320 text
+= ' nr_channels: '+str(self
.memory_nr_channels
)+'\n'
321 text
+= ' size: '+str(self
.memory_size
)+'\n'
322 text
+= 'hypervisor:\n'
323 text
+= self
.hypervisor
.to_text()
325 text
+= self
.os
.to_text()
326 text
+= 'resource topology:\n'
327 text
+= ' nr_nodes: '+ str(len(self
.nodes
))+'\n'
329 for node_k
, node_v
in self
.nodes
.iteritems():
330 text
+= ' node'+str(node_k
)+':\n'
331 text
+= node_v
.to_text()
335 return yaml
.load(self
.to_text())
339 self
.id_
= None #Integer. Node id. Unique in the system
340 self
.processor
= ProcessorNode() #Information about the processor in the node
341 self
.memory
= MemoryNode() #Information about the memory in the node
342 self
.nic_list
= list() #List of Nic() containing information about the nics associated to the node
343 self
.ports_list
= list() #List containing all network ports in the node. This is used to avoid having defined multiple times the same port in the system
345 def get_processor_info(self
):
346 """Gets the processor information. Returns (processor_family, processor_manufacturer, processor_version, processor_features)"""
347 return self
.processor
.get_info()
349 def get_memory_info(self
):
350 """Gets the memory information. Returns (memory_freq, memory_nr_channels, memory_type, memory_size)"""
351 return self
.memory
.get_info()
353 # def set(self, *args):
354 # """Sets the node information. Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
356 # processor = args[0]
360 # processor = args[0]
365 # return (False, 'Wrong number of elements calling Node().set()')
367 def set(self
, processor
, memory
, nic_list
):
368 (status
, return_code
) = self
.processor
.assign(processor
)
370 return (status
, return_code
)
372 self
.id_
= processor
.id_
374 (status
, return_code
) = self
.memory
.assign(memory
)
376 return (status
, return_code
)
380 if not isinstance(nic
,Nic
):
381 return (False, 'The nics must be of type Nic')
382 self
.nic_list
.append(nic
)
383 for port_key
in nic
.ports
.iterkeys():
384 if port_key
in self
.ports_list
:
385 return (False, 'Network port '+port_key
+'defined multiple times in the same node')
386 self
.ports_list
.append(port_key
)
390 def assign(self
, node
):
391 """Sets the node information.
392 Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
394 processor
= node
.processor
396 nic_list
= node
.nic_list
397 (status
, return_code
) = self
.processor
.assign(processor
)
399 return (status
, return_code
)
401 self
.id_
= processor
.id_
403 (status
, return_code
) = self
.memory
.assign(memory
)
405 return (status
, return_code
)
409 if not isinstance(nic
,Nic
):
410 return (False, 'The nics must be of type Nic')
411 self
.nic_list
.append(nic
)
412 for port_key
in nic
.ports
.iterkeys():
413 if port_key
in self
.ports_list
:
414 return (False, 'Network port '+port_key
+'defined multiple times in the same node')
415 self
.ports_list
.append(port_key
)
417 return (True,warning_text
)
420 text
= ' id: '+str(self
.id_
)+'\n'
422 text
+= self
.processor
.to_text()
424 text
+= self
.memory
.to_text()
425 if len(self
.nic_list
) > 0:
428 for nic
in self
.nic_list
:
429 text
+= ' nic '+str(nic_index
)+':\n'
430 text
+= nic
.to_text()
434 class ProcessorNode():
435 #Definition of the possible values of processor variables
436 possible_features
= definitionsClass
.processor_possible_features
437 possible_manufacturers
= definitionsClass
.processor_possible_manufacturers
438 possible_families
= definitionsClass
.processor_possible_families
439 possible_versions
= definitionsClass
.processor_possible_versions
442 self
.id_
= None #Integer. Numeric identifier of the socket
443 self
.family
= None #Text. Family name of the processor
444 self
.manufacturer
= None #Text. Manufacturer of the processor
445 self
.version
= None #Text. Model version of the processor
446 self
.features
= list() #list. List of features offered by the processor
447 self
.cores
= list() #list. List of cores in the processor. In case of hyperthreading the coupled cores are expressed as [a,b]
448 self
.eligible_cores
= list()#list. List of cores that can be used
449 #self.decicated_cores
450 #self.shared_cores -> this should also contain information to know if cores are being used
452 def assign(self
, processor
):
453 """Sets the processor information.
454 Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
455 if not isinstance(processor
,ProcessorNode
):
456 return (False, 'The variable \'processor\' must be of class ProcessorNode')
458 self
.id_
= processor
.id_
459 self
.family
= processor
.family
460 self
.manufacturer
= processor
.manufacturer
461 self
.version
= processor
.version
462 self
.features
= processor
.features
463 self
.cores
= processor
.cores
464 self
.eligible_cores
= processor
.eligible_cores
468 def set(self
, id_
, family
, manufacturer
, version
, features
, cores
):
469 """Sets the processor information.
470 Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
473 if not isinstance(id_
,int):
474 return (False, 'The processor id_ must be of type int')
475 if not isinstance(family
,str):
476 return (False, 'The processor family must be of type str')
477 if not isinstance(manufacturer
,str):
478 return (False, 'The processor manufacturer must be of type str')
479 if not isinstance(version
,str):
480 return (False, 'The processor version must be of type str')
481 if not isinstance(features
,list):
482 return (False, 'The processor features must be of type list')
483 if not isinstance(cores
,list):
484 return (False, 'The processor cores must be of type list')
486 (self
.id_
, self
.family
, self
.manufacturer
, self
.version
) = (id_
, family
, manufacturer
, version
)
488 if not manufacturer
in self
.possible_manufacturers
:
489 warning_text
+= "processor manufacturer '%s' not among: %s\n" %(manufacturer
, str(self
.possible_manufacturers
))
490 if not family
in self
.possible_families
:
491 warning_text
+= "family '%s' not among: %s\n" % (family
, str(self
.possible_families
))
492 # if not version in self.possible_versions:
493 # warning_text += 'The version %s is not one of these: %s\n' % (version, str(self.possible_versions))
495 for feature
in features
:
496 if not feature
in self
.possible_features
:
497 warning_text
+= "processor feature '%s' not among: %s\n" % (feature
, str(self
.possible_versions
))
498 self
.features
.append(feature
)
500 for iterator
in sorted(cores
):
501 if not isinstance(iterator
,list) or not all(isinstance(x
, int) for x
in iterator
):
502 return (False, 'The cores list must be in the form of [[a,b],[c,d],...] where a,b,c,d are of type int')
503 self
.cores
.append(iterator
)
505 self
.set_eligible_cores()
507 return (True,warning_text
)
509 def set_eligible_cores(self
):
510 """Set the default eligible cores, this is all cores non used by the host operating system"""
512 for iterator
in self
.cores
:
514 self
.eligible_cores
.append(iterator
)
520 """Returns processor parameters (self.family, self.manufacturer, self.version, self.features)"""
521 return (self
.family
, self
.manufacturer
, self
.version
, self
.features
)
524 text
= ' id: '+str(self
.id_
)+'\n'
525 text
+= ' family: '+self
.family
+'\n'
526 text
+= ' manufacturer: '+self
.manufacturer
+'\n'
527 text
+= ' version: '+self
.version
+'\n'
528 text
+= ' features: '+str(self
.features
)+'\n'
529 text
+= ' cores: '+str(self
.cores
)+'\n'
530 text
+= ' eligible_cores: '+str(self
.eligible_cores
)+'\n'
535 self
.modules
= list() #List of MemoryModule(). List of all modules installed in the node
536 self
.nr_channels
= None #Integer. Number of modules installed in the node
537 self
.node_size
= None #Integer. Total size in KiB of memory installed in the node
538 self
.eligible_memory
= None #Integer. Size in KiB of eligible memory in the node
539 self
.hugepage_sz
= None #Integer. Size in KiB of hugepages
540 self
.hugepage_nr
= None #Integer. Number of hugepages allocated in the module
541 self
.eligible_hugepage_nr
= None #Integer. Number of eligible hugepages in the node
542 self
.type_
= None #Text. Type of memory modules. If modules have a different value keep it as None
543 self
.freq
= None #Integer. Frequency of the modules in MHz. If modules have a different value keep it as None
544 self
.module_size
= None #Integer. Size of the modules in KiB. If modules have a different value keep it as None
545 self
.form_factor
= None #Text. Form factor of the modules. If modules have a different value keep it as None
547 def assign(self
, memory_node
):
548 return self
.set(memory_node
.modules
, memory_node
.hugepage_sz
, memory_node
.hugepage_nr
)
550 def set(self
, modules
, hugepage_sz
, hugepage_nr
):
551 """Set the memory node information. hugepage_sz must be expressed in KiB.
552 Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
553 if not isinstance(modules
, list):
554 return (False, 'The modules must be a list of elements of class MemoryModule')
555 if not isinstance(hugepage_sz
,int):
556 return (False, 'The hugepage_sz variable must be an int expressing the size in KiB')
557 if not isinstance(hugepage_nr
,int):
558 return (False, 'The hugepage_nr variable must be of type int')
560 (self
.hugepage_sz
, self
.hugepage_nr
) = (hugepage_sz
, hugepage_nr
)
561 self
.node_size
= self
.nr_channels
= 0
563 different_type
= different_freq
= different_module_size
= different_form_factor
= False
564 prev_type
= prev_freq
= prev_module_size
= prev_form_factor
= None
565 for iterator
in modules
:
566 if not isinstance(iterator
,MemoryModule
):
567 return (False, 'The modules must be a list of elements of class MemoryModule')
568 self
.modules
.append(iterator
)
569 (self
.type_
, self
.freq
, self
.module_size
, self
.form_factor
) = (iterator
.type_
, iterator
.freq
, iterator
.size
, iterator
.form_factor
)
570 self
.node_size
+= self
.module_size
571 self
.nr_channels
+= 1
572 if prev_type
!= None and prev_type
!= self
.type_
:
573 different_type
= True
574 if prev_freq
!= None and prev_freq
!= self
.freq
:
575 different_freq
= True
576 if prev_module_size
!= None and prev_module_size
!= self
.module_size
:
577 different_module_size
= True
578 if prev_form_factor
!= None and prev_form_factor
!= self
.form_factor
:
579 different_form_factor
= True
580 (prev_type
, prev_freq
, prev_module_size
, prev_form_factor
) = (self
.type_
, self
.freq
, self
.module_size
, self
.form_factor
)
586 if different_module_size
:
587 self
.module_size
= None
588 if different_form_factor
:
589 self
.form_factor
= None
591 (return_value
, error_code
) = self
.set_eligible_memory()
593 return (return_value
, error_code
)
597 def set_eligible_memory(self
):
598 """Sets the default eligible_memory and eligible_hugepage_nr. This is all memory but 2GiB and all hugepages"""
599 self
.eligible_memory
= self
.node_size
- 2*1024*1024
600 if self
.eligible_memory
< 0:
601 return (False, "There is less than 2GiB of memory in the module")
603 self
.eligible_hugepage_nr
= self
.hugepage_nr
607 """Return memory information (self.freq, self.nr_channels, self.type_, self.node_size)"""
608 return (self
.freq
, self
.nr_channels
, self
.type_
, self
.node_size
, self
.hugepage_sz
)
611 text
= ' node_size: '+str(self
.node_size
)+'\n'
612 text
+= ' nr_channels: '+str(self
.nr_channels
)+'\n'
613 text
+= ' eligible_memory: '+str(self
.eligible_memory
)+'\n'
614 text
+= ' hugepage_sz: '+str(self
.hugepage_sz
)+'\n'
615 text
+= ' hugepage_nr: '+str(self
.hugepage_nr
)+'\n'
616 text
+= ' eligible_hugepage_nr: '+str(self
.eligible_hugepage_nr
)+'\n'
617 text
+= ' type: '+self
.type_
+'\n'
618 text
+= ' freq: '+str(self
.freq
)+'\n'
619 text
+= ' module_size: '+str(self
.module_size
)+'\n'
620 text
+= ' form_factor: '+self
.form_factor
+'\n'
621 text
+= ' modules details:\n'
622 for module
in self
.modules
:
623 text
+= module
.to_text()
626 class MemoryModule():
627 #Definition of the possible values of module variables
628 possible_types
= definitionsClass
.memory_possible_types
629 possible_form_factors
= definitionsClass
.memory_possible_form_factors
632 self
.locator
= None #Text. Name of the memory module
633 self
.type_
= None #Text. Type of memory module
634 self
.freq
= None #Integer. Frequency of the module in MHz
635 self
.size
= None #Integer. Size of the module in KiB
636 self
.form_factor
= None #Text. Form factor of the module
638 def set(self
, locator
, type_
, freq
, size
, form_factor
):
639 """Sets the memory module information.
640 Frequency must be expressed in MHz and size in KiB.
641 Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
643 if not isinstance(locator
, str):
644 return (False, "The type of the variable locator must be str")
645 if not isinstance(type_
, str):
646 return (False, "The type of the variable type_ must be str")
647 if not isinstance(form_factor
, str):
648 return (False, "The type of the variable form_factor must be str")
649 if not isinstance(freq
, int):
650 return (False, "The type of the variable freq must be int")
651 if not isinstance(size
, int):
652 return (False, "The type of the variable size must be int")
654 if not form_factor
in self
.possible_form_factors
:
655 warning_text
+= "memory form_factor '%s' not among: %s\n" %(form_factor
, str(self
.possible_form_factors
))
656 if not type_
in self
.possible_types
:
657 warning_text
+= "memory type '%s' not among: %s\n" %(type_
, str(self
.possible_types
))
659 (self
.locator
, self
.type_
, self
.freq
, self
.size
, self
.form_factor
) = (locator
, type_
, freq
, size
, form_factor
)
660 return (True, warning_text
)
663 text
= ' '+self
.locator
+':\n'
664 text
+= ' type: '+self
.type_
+'\n'
665 text
+= ' freq: '+str(self
.freq
)+'\n'
666 text
+= ' size: '+str(self
.size
)+'\n'
667 text
+= ' form factor: '+self
.form_factor
+'\n'
672 self
.model
= None #Text. Model of the nic
673 self
.ports
= dict() #Dictionary of ports. Keys are the port name, value are Port() elements
675 def set_model(self
, model
):
676 """Sets the model of the nic. Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
677 if not isinstance(model
,str):
678 return (False, 'The \'model\' must be of type str')
683 def add_port(self
, port
):
684 """Adds a port to the nic. Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
685 if not isinstance(port
,Port
):
686 return (False, 'The \'port\' must be of class Port')
688 # port_id = str(port.pci_device_id[0])+':'+str(port.pci_device_id[1])+':'+str(port.pci_device_id[2])+'.'+str(port.pci_device_id[3])
690 # port_id = port.name
691 port_id
= port
.pci_device_id
693 if port_id
in self
.ports
:
694 return (False, 'The \'port\' '+port
.pci_device_id
+' is duplicated in the nic')
695 # return (False, 'The \'port\' is duplicated in the nic')
697 self
.ports
[port_id
] = port
701 text
= ' model: '+ str(self
.model
)+'\n'
702 text
+= ' ports: '+'\n'
703 for key
,port
in self
.ports
.iteritems():
704 text
+= ' "'+key
+'":'+'\n'
705 text
+= port
.to_text()
710 self
.name
= None #Text. Port name
711 self
.virtual
= None #Boolean. States if the port is a virtual function
712 self
.enabled
= None #Boolean. States if the port is enabled
713 self
.eligible
= None #Boolean. States if the port is eligible
714 self
.speed
= None #Integer. Indicates the speed in Mbps
715 self
.available_bw
= None #Integer. BW in Mbps that is available.
716 self
.mac
= None #list. Indicates the mac address of the port as a list in format ['XX','XX','XX','XX','XX','XX']
717 self
.pci_device_id_split
= None #list. Indicates the pci address of the port as a list in format ['XXXX','XX','XX','X']
718 self
.pci_device_id
= None
719 self
.PF_pci_device_id
= None
721 # def set(self, name, virtual, enabled, speed, mac, pci_device_id, pci_device_id_split):
722 # """Sets the port information. The variable speed indicates the speed in Mbps. Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
723 # if not isinstance(name,str):
724 # return (False, 'The variable \'name\' must be of type str')
725 # if not isinstance(virtual,bool):
726 # return (False, 'The variable \'virtual\' must be of type bool')
727 # if not isinstance(enabled,bool):
728 # return (False, 'The variable \'enabled\' must be of type bool')
729 # if not isinstance(enabled,bool):
730 # return (speed, 'The variable \'speed\' must be of type int')
731 # if not isinstance(mac, list) and not isinstance(mac,NoneType):
732 # return (False, 'The variable \'enabled\' must be of type list indicating the mac address in format [\'XXXX\',\'XX\',\'XX\',\'X\'] or NoneType')
733 # if not isinstance(pci_device_id_split, list) or len(pci_device_id_split) != 4:
734 # return (False, 'The variable \'pci_device_id_split\' must be of type list, indicating the pci address in format [\'XX\',\'XX\',\'XX\',\'XX\',\'XX\',\'XX\']')
736 # expected_len = [4,2,2,1]
738 # for iterator in pci_device_id_split:
739 # if not isinstance(iterator,str) or not iterator.isdigit() or len(iterator) != expected_len[index]:
740 # return (False, 'The variable \'pci_device_id_split\' must be of type list, indicating the pci address in format [\'XX\',\'XX\',\'XX\',\'XX\',\'XX\',\'XX\']')
743 # if not isinstance(mac,NoneType):
744 # for iterator in mac:
745 # if not isinstance(iterator,str) or not iterator.isalnum() or len(iterator) != 2:
746 # return (False, 'The variable \'enabled\' must be of type list indicating the mac address in format [\'XXXX\',\'XX\',\'XX\',\'X\'] or NoneType')
748 # #By default only virtual ports are eligible
749 # # (self.name, self.virtual, self.enabled, self.eligible, self.available_bw, self.speed, self.mac, self.pci_device_id, self.pci_device_id_split) = (name, virtual, enabled, virtual, speed, speed, mac, pci_device_id, pci_device_id_split)
750 # (self.name, self.virtual, self.enabled, self.eligible, self.available_bw, self.mac, self.pci_device_id, self.pci_device_id_split) = (name, virtual, enabled, virtual, speed, mac, pci_device_id, pci_device_id_split)
753 text
= ' pci: "'+ str(self
.pci_device_id
)+'"\n'
754 text
+= ' virtual: '+ str(self
.virtual
)+'\n'
756 text
+= ' PF_pci_id: "'+self
.PF_pci_device_id
+'"\n'
757 text
+= ' eligible: '+ str(self
.eligible
)+'\n'
758 text
+= ' enabled: '+str(self
.enabled
)+'\n'
759 text
+= ' speed: '+ str(self
.speed
)+'\n'
760 text
+= ' available bw: '+ str(self
.available_bw
)+'\n'
761 text
+= ' mac: '+ str(self
.mac
)+'\n'
762 text
+= ' source_name: '+ str(self
.name
)+'\n'
766 #Definition of the possible values of hypervisor variables
767 possible_types
= definitionsClass
.hypervisor_possible_types
768 possible_domain_types
= definitionsClass
.hypervisor_possible_domain_types
771 self
.type_
= None #Text. Hypervisor type_
772 self
.version
= None #int. Hypervisor version
773 self
.lib_version
= None #int. Libvirt version used to compile hypervisor
774 self
.domains
= list() #list. List of all the available domains
776 def set(self
, hypervisor
, version
, lib_version
, domains
):
778 if not isinstance(hypervisor
,str):
779 return (False, 'The variable type_ must be of type str')
780 if not isinstance(version
,int):
781 return (False, 'The variable version must be of type int')
782 if not isinstance(lib_version
,int):
783 return (False, 'The library version must be of type int')
784 if not isinstance(domains
,list):
785 return (False, 'Domains must be a list of the possible domains as str')
787 if not hypervisor
in self
.possible_types
:
788 warning_text
+= "Hyperpivor '%s' not among: %s\n" % (hypervisor
, str(self
.possible_types
))
790 valid_domain_found
= False
791 for domain
in domains
:
792 if not isinstance(domain
,str):
793 return (False, 'Domains must be a list of the possible domains as str')
794 if domain
in self
.possible_domain_types
:
795 valid_domain_found
= True
796 self
.domains
.append(domain
)
798 if not valid_domain_found
:
799 warning_text
+= 'No valid domain found among: %s\n' % str(self
.possible_domain_types
)
802 (self
.version
, self
.lib_version
, self
.type_
) = (version
, lib_version
, hypervisor
)
803 return (True, warning_text
)
805 def assign(self
, hypervisor
):
806 (self
.version
, self
.lib_version
, self
.type_
) = (hypervisor
.version
, hypervisor
.lib_version
, hypervisor
.type_
)
807 for domain
in hypervisor
.domains
:
808 self
.domains
.append(domain
)
812 text
= ' type: '+self
.type_
+'\n'
813 text
+= ' version: '+str(self
.version
)+'\n'
814 text
+= ' libvirt version: '+ str(self
.lib_version
)+'\n'
815 text
+= ' domains: '+str(self
.domains
)+'\n'
819 #Definition of the possible values of os variables
820 possible_id
= definitionsClass
.os_possible_id
821 possible_types
= definitionsClass
.os_possible_types
822 possible_architectures
= definitionsClass
.os_possible_architectures
825 self
.id_
= None #Text. Identifier of the OS. Formed by <Distibutor ID>-<Release>-<Codename>. In linux this can be obtained using lsb_release -a
826 self
.type_
= None #Text. Type of operating system
827 self
.bit_architecture
= None #Integer. Architecture
829 def set(self
, id_
, type_
, bit_architecture
):
831 if not isinstance(type_
,str):
832 return (False, 'The variable type_ must be of type str')
833 if not isinstance(id_
,str):
834 return (False, 'The variable id_ must be of type str')
835 if not isinstance(bit_architecture
,str):
836 return (False, 'The variable bit_architecture must be of type str')
838 if not type_
in self
.possible_types
:
839 warning_text
+= "os type '%s' not among: %s\n" %(type_
, str(self
.possible_types
))
840 if not id_
in self
.possible_id
:
841 warning_text
+= "os release '%s' not among: %s\n" %(id_
, str(self
.possible_id
))
842 if not bit_architecture
in self
.possible_architectures
:
843 warning_text
+= "os bit_architecture '%s' not among: %s\n" % (bit_architecture
, str(self
.possible_architectures
))
845 (self
.id_
, self
.type_
, self
.bit_architecture
) = (id_
, type_
, bit_architecture
)
846 return (True, warning_text
)
849 (self
.id_
, self
.type_
, self
.bit_architecture
) = (os
.id_
, os
.type_
, os
.bit_architecture
)
853 text
= ' id: '+self
.id_
+'\n'
854 text
+= ' type: '+self
.type_
+'\n'
855 text
+= ' bit_architecture: '+self
.bit_architecture
+'\n'
858 def get_hostname(virsh_conn
):
859 return virsh_conn
.getHostname().rstrip('\n')
861 def get_hugepage_size(ssh_conn
):
862 command
= 'sudo hugeadm --page-sizes'
863 # command = 'hugeadm --page-sizes-all'
864 (_
, stdout
, stderr
) = ssh_conn
.exec_command(command
)
865 error
= stderr
.read()
867 raise paramiko
.ssh_exception
.SSHException(command
+' : '+ error
)
873 def get_hugepage_nr(ssh_conn
,hugepage_sz
, node_id
):
874 command
= 'cat /sys/devices/system/node/node'+str(node_id
)+'/hugepages/hugepages-'+str(hugepage_sz
/1024)+'kB/nr_hugepages'
875 (_
, stdout
, _
) = ssh_conn
.exec_command(command
)
877 #text = stdout.read()
882 value
=int(stdout
.read())
887 def get_memory_information(ssh_conn
, virsh_conn
, memory_nodes
):
889 tree
=ElementTree
.fromstring(virsh_conn
.getSysinfo(0))
891 node_id
= 0 #TODO revise. Added for allowing VM as compute hosts
892 for target
in tree
.findall("memory_device"):
893 locator_f
= size_f
= freq_f
= type_f
= formfactor_f
= False
894 locator_f
= True #TODO revise. Added for allowing VM as compute hosts
895 module_form_factor
= ""
896 for entry
in target
.findall("entry"):
897 if entry
.get("name") == 'size':
899 size_split
= entry
.text
.split(' ')
900 if size_split
[1] == 'MB':
901 module_size
= int(size_split
[0]) * 1024 * 1024
902 elif size_split
[1] == 'GB':
903 module_size
= int(size_split
[0]) * 1024 * 1024 * 1024
904 elif size_split
[1] == 'KB':
905 module_size
= int(size_split
[0]) * 1024
907 module_size
= int(size_split
[0])
909 elif entry
.get("name") == 'speed':
911 freq_split
= entry
.text
.split(' ')
912 if freq_split
[1] == 'MHz':
913 module_freq
= int(freq_split
[0]) * 1024 * 1024
914 elif freq_split
[1] == 'GHz':
915 module_freq
= int(freq_split
[0]) * 1024 * 1024 * 1024
916 elif freq_split
[1] == 'KHz':
917 module_freq
= int(freq_split
[0]) * 1024
919 elif entry
.get("name") == 'type':
921 module_type
= entry
.text
923 elif entry
.get("name") == 'form_factor':
925 module_form_factor
= entry
.text
926 #TODO revise. Commented for allowing VM as compute hosts
927 # elif entry.get("name") == 'locator' and not locator_f:
928 # # other case, it is obtained by bank_locator that we give priority to
929 # locator = entry.text
930 # pos = locator.find(module_form_factor)
931 # if module_form_factor == locator[0:len(module_form_factor) ]:
932 # pos = len(module_form_factor) +1
935 # if locator[pos] in "ABCDEFGH":
937 # node_id = ord(locator[pos])-ord('A')
938 # #print entry.text, node_id
940 # elif entry.get("name") == 'bank_locator':
941 # locator = entry.text
942 # pos = locator.find("NODE ")
943 # if pos >= 0 and len(locator)>pos+5:
944 # if locator[pos+5] in ("01234567"): #len("NODE ") is 5
945 # node_id = int(locator[pos+5])
949 #When all module fields have been found add a new module to the list
950 if locator_f
and size_f
and freq_f
and type_f
and formfactor_f
:
951 #If the memory node has not yet been created create it
952 if node_id
not in memory_dict
:
953 memory_dict
[node_id
] = []
955 #Add a new module to the memory node
956 module
= MemoryModule()
957 #TODO revise. Changed for allowing VM as compute hosts
958 (return_status
, code
) = module
.set('NODE %d' % node_id
, module_type
, module_freq
, module_size
, module_form_factor
)
959 #(return_status, code) = module.set(locator, module_type, module_freq, module_size, module_form_factor)
960 if not return_status
:
961 return (return_status
, code
)
962 memory_dict
[node_id
].append(module
)
963 if code
not in warning_text
:
965 node_id
+= 1 #TODO revise. Added for allowing VM as compute hosts
968 #Hugepage size is constant for all nodes
969 hugepage_sz
= get_hugepage_size(ssh_conn
)
970 for node_id
, modules
in memory_dict
.iteritems():
971 memory_node
= MemoryNode()
972 memory_node
.set(modules
, hugepage_sz
, get_hugepage_nr(ssh_conn
,hugepage_sz
, node_id
))
973 memory_nodes
[node_id
] = memory_node
975 return (True, warning_text
)
977 def get_cpu_topology_ht(ssh_conn
, topology
):
978 command
= 'cat /proc/cpuinfo'
979 (_
, stdout
, stderr
) = ssh_conn
.exec_command(command
)
980 error
= stderr
.read()
982 raise paramiko
.ssh_exception
.SSHException(command
+' : '+ error
)
988 for line
in stdout
.readlines():
989 if len(line
.strip()) != 0:
990 name
, value
= line
.split(":", 1)
991 core_lines
[name
.strip()] = value
.strip()
993 core_details
.append(core_lines
)
996 for core
in core_details
:
997 for field
in ["processor", "core id", "physical id"]:
998 if field
not in core
:
999 return(False,'Error getting '+field
+' value from /proc/cpuinfo')
1000 core
[field
] = int(core
[field
])
1002 if core
["core id"] not in cores
:
1003 cores
.append(core
["core id"])
1004 if core
["physical id"] not in sockets
:
1005 sockets
.append(core
["physical id"])
1006 key
= (core
["physical id"], core
["core id"])
1007 if key
not in core_map
:
1009 core_map
[key
].append(core
["processor"])
1012 hyperthreaded_cores
= list()
1014 hyperthreaded_cores
.append(core_map
[(s
,c
)])
1015 topology
[s
] = hyperthreaded_cores
1019 def get_processor_information(ssh_conn
, vish_conn
, processors
):
1021 #Processor features are the same for all processors
1022 #TODO (at least using virsh capabilities)nr_numa_nodes
1023 capabilities
= list()
1024 tree
=ElementTree
.fromstring(vish_conn
.getCapabilities())
1025 for target
in tree
.findall("host/cpu/feature"):
1026 if target
.get("name") == 'pdpe1gb':
1027 capabilities
.append('lps')
1028 elif target
.get("name") == 'dca':
1029 capabilities
.append('dioc')
1030 elif target
.get("name") == 'vmx' or target
.get("name") == 'svm':
1031 capabilities
.append('hwsv')
1032 elif target
.get("name") == 'ht':
1033 capabilities
.append('ht')
1035 target
= tree
.find("host/cpu/arch")
1036 if target
.text
== 'x86_64' or target
.text
== 'amd64':
1037 capabilities
.append('64b')
1039 command
= 'cat /proc/cpuinfo | grep flags'
1040 (_
, stdout
, stderr
) = ssh_conn
.exec_command(command
)
1041 error
= stderr
.read()
1043 raise paramiko
.ssh_exception
.SSHException(command
+' : '+ error
)
1044 line
= stdout
.readline()
1045 if 'ept' in line
or 'npt' in line
:
1046 capabilities
.append('tlbps')
1048 #Find out if IOMMU is enabled
1049 command
= 'dmesg |grep -e Intel-IOMMU'
1050 (_
, stdout
, stderr
) = ssh_conn
.exec_command(command
)
1051 error
= stderr
.read()
1053 raise paramiko
.ssh_exception
.SSHException(command
+' : '+ error
)
1054 if 'enabled' in stdout
.read():
1055 capabilities
.append('iommu')
1058 command
= 'dmesg |grep -e AMD-Vi'
1059 (_
, stdout
, stderr
) = ssh_conn
.exec_command(command
)
1060 error
= stderr
.read()
1062 raise paramiko
.ssh_exception
.SSHException(command
+' : '+ error
)
1063 if len(stdout
.read()) > 0:
1064 capabilities
.append('iommu')
1066 #-----------------------------------------------------------
1068 #In case hyperthreading is active it is necessary to determine cpu topology using /proc/cpuinfo
1069 if 'ht' in capabilities
:
1070 (return_status
, code
) = get_cpu_topology_ht(ssh_conn
, topology
)
1071 if not return_status
:
1072 return (return_status
, code
)
1073 warning_text
+= code
1075 #Otherwise it is possible to do it using virsh capabilities
1077 for target
in tree
.findall("host/topology/cells/cell"):
1078 socket_id
= int(target
.get("id"))
1079 topology
[socket_id
] = list()
1080 for cpu
in target
.findall("cpus/cpu"):
1081 topology
[socket_id
].append(int(cpu
.get("id")))
1083 #-----------------------------------------------------------
1084 #Create a dictionary with the information of all processors
1085 #p_fam = p_man = p_ver = None
1086 tree
=ElementTree
.fromstring(vish_conn
.getSysinfo(0))
1087 #print vish_conn.getSysinfo(0)
1088 #return (False, 'forces error for debuging')
1090 socket_id
= -1 #in case we can not determine the socket_id we assume incremental order, starting by 0
1091 for target
in tree
.findall("processor"):
1094 #Get processor id, family, manufacturer and version
1095 for entry
in target
.findall("entry"):
1096 if entry
.get("name") == "status":
1097 if entry
.text
[0:11] == "Unpopulated":
1099 elif entry
.get("name") == 'socket_destination':
1100 socket_text
= entry
.text
1101 if socket_text
.startswith('CPU'):
1102 socket_text
= socket_text
.strip('CPU')
1103 socket_text
= socket_text
.strip() #removes trailing spaces
1104 if socket_text
.isdigit() and int(socket_text
)<9 and int(socket_text
)>0:
1105 socket_id
= int(socket_text
) - 1
1107 elif entry
.get("name") == 'family':
1110 elif entry
.get("name") == 'manufacturer':
1111 manufacturer
= entry
.text
1113 elif entry
.get("name") == 'version':
1114 version
= entry
.text
.strip()
1117 return (False, 'Error. Not all expected fields could be found in processor')
1119 #Create and fill processor structure
1121 continue #avoid inconsistence of some machines where more socket detected than
1122 processor
= ProcessorNode()
1123 (return_status
, code
) = processor
.set(socket_id
, family
, manufacturer
, version
, capabilities
, topology
[socket_id
])
1124 if not return_status
:
1125 return (return_status
, code
)
1126 if code
not in warning_text
:
1127 warning_text
+= code
1129 #Add processor to the processors dictionary
1130 processors
[socket_id
] = processor
1132 return (True, warning_text
)
1134 def get_nic_information(ssh_conn
, virsh_conn
, nic_topology
):
1136 #Get list of net devices
1137 net_devices
= virsh_conn
.listDevices('net',0)
1138 print virsh_conn
.listDevices('net',0)
1139 for device
in net_devices
:
1141 #Get the XML descriptor of the device:
1142 net_XML
= ElementTree
.fromstring(virsh_conn
.nodeDeviceLookupByName(device
).XMLDesc(0))
1143 #print "net_XML:" , net_XML
1145 parent
= net_XML
.find('parent')
1147 print 'No parent was found in XML for device '+device
1148 #Error. continue?-------------------------------------------------------------
1150 if parent
.text
== 'computer':
1152 if not parent
.text
.startswith('pci_'):
1153 print device
+ ' parent is neither computer nor pci'
1154 #Error. continue?-------------------------------------------------------------
1156 interface
= net_XML
.find('capability/interface').text
1157 mac
= net_XML
.find('capability/address').text
1160 pci_XML
= ElementTree
.fromstring(virsh_conn
.nodeDeviceLookupByName(parent
.text
).XMLDesc(0))
1163 name
= pci_XML
.find('name').text
.split('_')
1164 pci
= name
[1]+':'+name
[2]+':'+name
[3]+'.'+name
[4]
1166 #If slot == 0 it is a PF, otherwise it is a VF
1167 capability
= pci_XML
.find('capability')
1168 if capability
.get('type') != 'pci':
1169 print device
+ 'Capability is not of type pci in '+parent
.text
1170 #Error. continue?-------------------------------------------------------------
1172 slot
= capability
.find('slot').text
1173 bus
= capability
.find('bus').text
1175 numa_
= capability
.find('numa')
1177 node_id
= numa_
.get('node');
1178 if node_id
!= None: node_id
=int(node_id
)
1179 if slot
== None or bus
== None:
1180 print device
+ 'Bus and slot not detected in '+parent
.text
1181 #Error. continue?-------------------------------------------------------------
1184 # print ElementTree.tostring(pci_XML)
1186 capability_pf
= capability
.find('capability')
1187 if capability_pf
.get('type') != 'phys_function':
1188 print 'physical_function not found in VF '+parent
.text
1189 #Error. continue?-------------------------------------------------------------
1191 PF_pci
= capability_pf
.find('address').attrib
1192 PF_pci_text
= PF_pci
['domain'].split('x')[1]+':'+PF_pci
['bus'].split('x')[1]+':'+PF_pci
['slot'].split('x')[1]+'.'+PF_pci
['function'].split('x')[1]
1197 #Obtain node for the port
1199 node_id
= int(bus
)>>6
1200 #print "node_id:", node_id
1202 #Only for non virtual interfaces: Obtain speed and if link is detected (this must be done using ethtool)
1204 command
= 'sudo ethtool '+interface
+' | grep -e Speed -e "Link detected"'
1205 (_
, stdout
, stderr
) = ssh_conn
.exec_command(command
)
1206 error
= stderr
.read()
1208 print 'Error running '+command
+'\n'+error
1209 #Error. continue?-------------------------------------------------------------
1211 for line
in stdout
.readlines():
1212 line
= line
.strip().rstrip('\n').split(': ')
1213 if line
[0] == 'Speed':
1214 if line
[1].endswith('Mb/s'):
1215 speed
= int(line
[1].split('M')[0])*int(1e6
)
1216 elif line
[1].endswith('Gb/s'):
1217 speed
= int(line
[1].split('G')[0])*int(1e9
)
1218 elif line
[1].endswith('Kb/s'):
1219 speed
= int(line
[1].split('K')[0])*int(1e3
)
1221 #the interface is listed but won't be used
1223 elif line
[0] == 'Link detected':
1224 if line
[1] == 'yes':
1229 print 'Unnexpected output of command '+command
+':'
1231 #Error. continue?-------------------------------------------------------------
1234 if not node_id
in nic_topology
:
1235 nic_topology
[node_id
] = list()
1236 #With this implementation we make the RAD with only one nic per node and this nic has all ports, TODO: change this by including parent information of PF
1237 nic_topology
[node_id
].append(Nic())
1239 #Load the appropriate nic
1240 nic
= nic_topology
[node_id
][0]
1242 #Create a new port and fill it
1244 port
.name
= interface
1245 port
.virtual
= virtual
1248 port
.available_bw
= 0
1249 port
.PF_pci_device_id
= PF_pci_text
1251 port
.available_bw
= speed
1253 port
.enabled
= False
1255 port
.enabled
= enabled
1257 port
.eligible
= virtual
#Only virtual ports are eligible
1259 port
.pci_device_id
= pci
1260 port
.pci_device_id_split
= name
[1:]
1262 #Save the port information
1265 print 'Error: '+str(e
)
1267 #set in vitual ports if they are enabled
1268 for nic
in nic_topology
.itervalues():
1269 for port
in nic
[0].ports
.itervalues():
1270 # print port.pci_device_id
1272 enabled
= nic
[0].ports
.get(port
.PF_pci_device_id
)
1274 return(False, 'The PF '+port
.PF_pci_device_id
+' (VF '+port
.pci_device_id
+') is not present in ports dict')
1275 #Only if the PF is enabled the VF can be enabled
1276 if nic
[0].ports
[port
.PF_pci_device_id
].enabled
:
1279 port
.enabled
= False
1281 return (True, warning_text
)
1283 def get_nic_information_old(ssh_conn
, nic_topology
):
1284 command
= 'lstopo-no-graphics --of xml'
1285 (_
, stdout
, stderr
) = ssh_conn
.exec_command(command
)
1286 error
= stderr
.read()
1288 raise paramiko
.ssh_exception
.SSHException(command
+' : '+ error
)
1289 tree
=ElementTree
.fromstring(stdout
.read())
1290 for target
in tree
.findall("object/object"):
1292 if target
.get("type") != "NUMANode":
1294 node_id
= int(target
.get("os_index"))
1295 nic_topology
[node_id
] = list()
1297 #find nics in numa node
1298 for entry
in target
.findall("object/object"):
1299 if entry
.get("type") != 'Bridge':
1301 nic_name
= entry
.get("name")
1306 for pcidev
in entry
.findall("object"):
1307 if pcidev
.get("type") != 'PCIDev':
1309 enabled
= speed
= mac
= pci_busid
= None
1311 model
= pcidev
.get("name")
1313 if 'Virtual' in model
:
1315 pci_busid
= pcidev
.get("pci_busid")
1316 for osdev
in pcidev
.findall("object"):
1317 name
= osdev
.get("name")
1318 for info
in osdev
.findall("info"):
1319 if info
.get("name") != 'Address':
1321 mac
= info
.get("value")
1322 #get the port speed and status
1323 command
= 'sudo ethtool '+name
1324 (_
, stdout
, stderr
) = ssh_conn
.exec_command(command
)
1325 error
= stderr
.read()
1327 return (False, 'Error obtaining '+name
+' information: '+error
)
1328 ethtool
= stdout
.read()
1329 if '10000baseT/Full' in ethtool
:
1331 elif '1000baseT/Full' in ethtool
:
1333 elif '100baseT/Full' in ethtool
:
1335 elif '10baseT/Full' in ethtool
:
1338 return (False, 'Speed not detected in '+name
)
1341 if 'Link detected: yes' in ethtool
:
1344 if speed
!= None and mac
!= None and pci_busid
!= None:
1345 mac
= mac
.split(':')
1346 pci_busid_split
= re
.split(':|\.', pci_busid
)
1347 #Fill the port information
1348 port
.set(name
, virtual
, enabled
, speed
, mac
, pci_busid
, pci_busid_split
)
1351 if len(nic
.ports
) > 0:
1354 nic
.set_model(model
)
1356 nic
.set_model(nic_name
)
1358 #Add it to the topology
1359 nic_topology
[node_id
].append(nic
)
1363 def get_os_information(ssh_conn
, os
):
1365 # command = 'lsb_release -a'
1366 # (stdin, stdout, stderr) = ssh_conn.exec_command(command)
1368 # for line in stdout.readlines():
1369 # line_split = re.split('\t| *', line.rstrip('\n'))
1370 # if line_split[0] == 'Distributor' and line_split[1] == 'ID:':
1371 # distributor = line_split[2]
1373 # elif line_split[0] == 'Release:':
1374 # release = line_split[1]
1376 # elif line_split[0] == 'Codename:':
1377 # codename = line_split[1]
1380 # return (False, 'It was not possible to obtain the OS id')
1381 # id_ = distributor+'-'+release+'-'+codename
1384 command
= 'cat /etc/redhat-release'
1385 (_
, stdout
, _
) = ssh_conn
.exec_command(command
)
1386 id_text
= stdout
.read()
1389 command
= 'lsb_release -d -s'
1390 (_
, stdout
, _
) = ssh_conn
.exec_command(command
)
1391 id_text
= stdout
.read()
1393 raise paramiko
.ssh_exception
.SSHException("Can not determinte release neither with 'lsb_release' nor with 'cat /etc/redhat-release'")
1394 id_
= id_text
.rstrip('\n')
1396 command
= 'uname -o'
1397 (_
, stdout
, stderr
) = ssh_conn
.exec_command(command
)
1398 error
= stderr
.read()
1400 raise paramiko
.ssh_exception
.SSHException(command
+' : '+ error
)
1401 type_
= stdout
.read().rstrip('\n')
1403 command
= 'uname -i'
1404 (_
, stdout
, stderr
) = ssh_conn
.exec_command(command
)
1405 error
= stderr
.read()
1407 raise paramiko
.ssh_exception
.SSHException(command
+' : '+ error
)
1408 bit_architecture
= stdout
.read().rstrip('\n')
1410 (return_status
, code
) = os
.set(id_
, type_
, bit_architecture
)
1411 if not return_status
:
1412 return (return_status
, code
)
1413 warning_text
+= code
1414 return (True, warning_text
)
1416 def get_hypervisor_information(virsh_conn
, hypervisor
):
1417 type_
= virsh_conn
.getType().rstrip('\n')
1418 version
= virsh_conn
.getVersion()
1419 lib_version
= virsh_conn
.getLibVersion()
1422 tree
=ElementTree
.fromstring(virsh_conn
.getCapabilities())
1423 for target
in tree
.findall("guest"):
1424 os_type
= target
.find("os_type").text
1425 #We only allow full virtualization
1426 if os_type
!= 'hvm':
1428 wordsize
= int(target
.find('arch/wordsize').text
)
1430 for domain
in target
.findall("arch/domain"):
1431 domains
.append(domain
.get("type"))
1433 (return_status
, code
) = hypervisor
.set(type_
, version
, lib_version
, domains
)
1434 if not return_status
:
1435 return (return_status
, code
)
1438 class RADavailableResourcesClass(RADclass
):
1439 def __init__(self
, resources
):
1440 """Copy resources from the RADclass (server resources not taking into account resources used by VMs"""
1442 self
.reserved
= dict() #Dictionary of reserved resources for a server. Key are VNFC names and values RADreservedResources
1443 self
.cores_consumption
= None #Dictionary of cpu consumption. Key is the cpu and the value is
1445 self
.machine
= resources
.machine
1446 self
.user
= resources
.user
1447 self
.password
= resources
.password
1448 self
.name
= resources
.name
1449 self
.nr_processors
= resources
.nr_processors
1450 self
.processor_family
= resources
.processor_family
1451 self
.processor_manufacturer
= resources
.processor_manufacturer
1452 self
.processor_version
= resources
.processor_version
1453 self
.processor_features
= resources
.processor_features
1454 self
.memory_type
= resources
.memory_type
1455 self
.memory_freq
= resources
.memory_freq
1456 self
.memory_nr_channels
= resources
.memory_nr_channels
1457 self
.memory_size
= resources
.memory_size
1458 self
.memory_hugepage_sz
= resources
.memory_hugepage_sz
1459 self
.hypervisor
= Hypervisor()
1460 self
.hypervisor
.assign(resources
.hypervisor
)
1462 self
.os
.assign(resources
.os
)
1464 for node_k
, node_v
in resources
.nodes
.iteritems():
1465 self
.nodes
[node_k
] = Node()
1466 self
.nodes
[node_k
].assign(node_v
)
1469 def _get_cores_consumption_warnings(self
):
1470 """Returns list of warning strings in case warnings are generated.
1471 In case no warnings are generated the return value will be an empty list"""
1473 #Get the cores consumption
1474 (return_status
, code
) = get_ssh_connection(self
.machine
, self
.user
, self
.password
)
1475 if not return_status
:
1476 return (return_status
, code
)
1478 command
= 'mpstat -P ALL 1 1 | grep Average | egrep -v CPU\|all'
1479 (_
, stdout
, stderr
) = ssh_conn
.exec_command(command
)
1480 error
= stderr
.read()
1482 return (False, error
)
1484 self
.cores_consumption
= dict()
1485 for line
in stdout
.readlines():
1486 cpu_usage_split
= re
.split('\t| *', line
.rstrip('\n'))
1487 usage
= 100 *(1 - float(cpu_usage_split
[10]))
1489 self
.cores_consumption
[int(cpu_usage_split
[1])] = usage
1491 #Check if any core marked as available in the nodes has cpu_usage > 0
1492 for _
, node_v
in self
.nodes
.iteritems():
1493 cores
= node_v
.processor
.eligible_cores
1497 if core
in self
.cores_consumption
:
1498 warnings
.append('Warning: Core '+str(core
)+' is supposed to be idle but it is consuming '+str(self
.cores_consumption
[core
])+'%')
1500 if cpu
in self
.cores_consumption
:
1501 warnings
.append('Warning: Core '+str(core
)+' is supposed to be idle but it is consuming '+str(self
.cores_consumption
[cpu
])+'%')
1505 def reserved_to_text(self
):
1507 for VNFC_name
, VNFC_reserved
in self
.reserved
.iteritems():
1508 text
+= ' VNFC: '+str(VNFC_name
)+'\n'
1509 text
+= VNFC_reserved
.to_text()
1513 def obtain_usage(self
):
1515 #Iterate through nodes to get cores, eligible cores, memory and physical ports (save ports usage for next section)
1517 ports_usage
= dict()
1518 hugepage_size
= dict()
1519 for node_k
, node_v
in self
.nodes
.iteritems():
1521 ports_usage
[node_k
] = dict()
1522 eligible_cores
= list()
1523 for pair
in node_v
.processor
.eligible_cores
:
1524 if isinstance(pair
, list):
1525 for element
in pair
:
1526 eligible_cores
.append(element
)
1528 eligible_cores
.append(pair
)
1529 node
['cpus'] = {'cores':node_v
.processor
.cores
,'eligible_cores':eligible_cores
}
1530 node
['memory'] = {'size':str(node_v
.memory
.node_size
/(1024*1024*1024))+'GB','eligible':str(node_v
.memory
.eligible_memory
/(1024*1024*1024))+'GB'}
1531 hugepage_size
[node_k
] = node_v
.memory
.hugepage_sz
1534 for nic
in node_v
.nic_list
:
1535 for port
in nic
.ports
.itervalues():
1536 if port
.enabled
and not port
.virtual
:
1537 ports
[port
.name
] = {'speed':str(port
.speed
/1000000000)+'G'}
1538 # print '*************** ',port.name,'speed',port.speed
1539 ports_usage
[node_k
][port
.name
] = 100 - int(100*float(port
.available_bw
)/float(port
.speed
))
1540 node
['ports'] = ports
1541 nodes
[node_k
] = node
1544 #Iterate through reserved section to get used cores, used memory and port usage
1547 #reserved_cores = list
1548 for node_k
in self
.nodes
.iterkeys():
1549 if not node_k
in cores
:
1550 cores
[node_k
] = list()
1552 for _
, reserved
in self
.reserved
.iteritems():
1553 if node_k
in reserved
.node_reserved_resources
:
1554 node_v
= reserved
.node_reserved_resources
[node_k
]
1555 cores
[node_k
].extend(node_v
.reserved_cores
)
1556 memory
[node_k
] += node_v
.reserved_hugepage_nr
* hugepage_size
[node_k
]
1559 for node_k
in self
.nodes
.iterkeys():
1561 for name
, usage
in ports_usage
[node_k
].iteritems():
1562 ports
[name
] = {'occupied':str(usage
)+'%'}
1563 # print '****************cores',cores
1564 # print '****************memory',memory
1565 occupation
[node_k
] = {'cores':cores
[node_k
],'memory':str(memory
[node_k
]/(1024*1024*1024))+'GB','ports':ports
}
1566 resp
['occupation'] = occupation
1570 class RADreservedResources():
1572 self
.node_reserved_resources
= dict() #dict. keys are the RAD nodes id, values are NodeReservedResources
1573 self
.mgmt_interface_pci
= None #pci in the VNF for the management interface
1574 self
.image
= None #Path in remote machine of the VNFC image
1576 def update(self
,reserved
):
1577 self
.image
= reserved
.image
1578 self
.mgmt_interface_pci
= reserved
.mgmt_interface_pci
1579 for k
,v
in reserved
.node_reserved_resources
.iteritems():
1580 if k
in self
.node_reserved_resources
.keys():
1581 return (False, 'Duplicated node entry '+str(k
)+' in reserved resources')
1582 self
.node_reserved_resources
[k
]=v
1587 text
= ' image: '+str(self
.image
)+'\n'
1588 for node_id
, node_reserved
in self
.node_reserved_resources
.iteritems():
1589 text
+= ' Node ID: '+str(node_id
)+'\n'
1590 text
+= node_reserved
.to_text()
1593 class NodeReservedResources():
1595 # reserved_shared_cores = None #list. List of all cores that the VNFC needs in shared mode #TODO Not used
1596 # reserved_memory = None #Integer. Amount of KiB needed by the VNFC #TODO. Not used since hugepages are used
1597 self
.reserved_cores
= list() #list. List of all cores that the VNFC uses
1598 self
.reserved_hugepage_nr
= 0 #Integer. Number of hugepages needed by the VNFC
1599 self
.reserved_ports
= dict() #dict. The key is the physical port pci and the value the VNFC port description
1600 self
.vlan_tags
= dict()
1601 self
.cpu_pinning
= None
1604 text
= ' cores: '+str(self
.reserved_cores
)+'\n'
1605 text
+= ' cpu_pinning: '+str(self
.cpu_pinning
)+'\n'
1606 text
+= ' hugepages_nr: '+str(self
.reserved_hugepage_nr
)+'\n'
1607 for port_pci
, port_description
in self
.reserved_ports
.iteritems():
1608 text
+= ' port: '+str(port_pci
)+'\n'
1609 text
+= port_description
.to_text()
1612 # def update(self,reserved):
1613 # self.reserved_cores = list(reserved.reserved_cores)
1614 # self.reserved_hugepage_nr = reserved.reserved_hugepage_nr
1615 # self.reserved_ports = dict(reserved.reserved_ports)
1616 # self.cpu_pinning = list(reserved.cpu_pinning)