1 # -*- coding: utf-8 -*-
4 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
5 # This file is part of openvim
8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
9 # not use this file except in compliance with the License. You may obtain
10 # a copy of the License at
12 # http://www.apache.org/licenses/LICENSE-2.0
14 # Unless required by applicable law or agreed to in writing, software
15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17 # License for the specific language governing permissions and limitations
20 # For those usages not covered by the Apache License, Version 2.0 please
21 # contact with: nfvlabs@tid.es
25 Implement the logic for obtaining compute nodes information
26 Resource Availability Descriptor
28 __author__
="Pablo Montes"
30 #TODO: remove warnings, remove unused things
32 from definitionsClass
import definitionsClass
33 from auxiliary_functions
import get_ssh_connection
35 from xml
.etree
import ElementTree
41 def getCredentials(creds
, data
):
42 """Used as a backup for libvirt.openAuth in order to provide password that came with data,
43 not used by the moment
45 print "RADclass:getCredentials", creds
, data
48 if cred
[0] == libvirt
.VIR_CRED_AUTHNAME
:
50 elif cred
[0] == libvirt
.VIR_CRED_PASSPHRASE
:
62 self
.nodes
= dict() #Dictionary of nodes. Keys are the node id, values are Node() elements
63 self
.nr_processors
= None #Integer. Number of processors in the system
64 self
.processor_family
= None #If all nodes have the same value equal them, otherwise keep as None
65 self
.processor_manufacturer
= None #If all nodes have the same value equal them, otherwise keep as None
66 self
.processor_version
= None #If all nodes have the same value equal them, otherwise keep as None
67 self
.processor_features
= None #If all nodes have the same value equal them, otherwise keep as None
68 self
.memory_type
= None #If all nodes have the same value equal them, otherwise keep as None
69 self
.memory_freq
= None #If all nodes have the same value equal them, otherwise keep as None
70 self
.memory_nr_channels
= None #If all nodes have the same value equal them, otherwise keep as None
71 self
.memory_size
= None #Integer. Sum of the memory in all nodes
72 self
.memory_hugepage_sz
= None
73 self
.hypervisor
= Hypervisor() #Hypervisor information
74 self
.os
= OpSys() #Operating system information
75 self
.ports_list
= list() #List containing all network ports in the node. This is used to avoid having defined multiple times the same port in the system
78 def obtain_RAD(self
, user
, password
, machine
):
79 """This function obtains the RAD information from the remote server.
80 It uses both a ssh and a libvirt connection.
81 It is desirable in future versions get rid of the ssh connection, but currently
82 libvirt does not provide all the needed information.
83 Returns (True, Warning) in case of success and (False, <error>) in case of error"""
86 #Get virsh and ssh connection
87 (return_status
, code
) = get_ssh_connection(machine
, user
, password
)
89 print 'RADclass.obtain_RAD() error:', code
90 return (return_status
, code
)
93 self
.connection_IP
= machine
94 #print "libvirt open pre"
95 virsh_conn
=libvirt
.open("qemu+ssh://"+user
+'@'+machine
+"/system")
96 #virsh_conn=libvirt.openAuth("qemu+ssh://"+user+'@'+machine+"/system",
97 # [[libvirt.VIR_CRED_AUTHNAME, libvirt.VIR_CRED_PASSPHRASE, libvirt.VIR_CRED_USERNAME], getCredentials, password],
99 #print "libvirt open after"
101 # #Set connection infomation
102 # (return_status, code) = self.set_connection_info(machine, user, password)
103 # if not return_status:
104 # return (return_status, 'Error in '+machine+': '+code)
107 machine_name
= get_hostname(virsh_conn
)
108 (return_status
, code
) = self
.set_name(machine_name
)
109 if not return_status
:
110 return (return_status
, 'Error at self.set_name in '+machine
+': '+code
)
113 #Get the server processors information
115 (return_status
, code
) = get_processor_information(ssh_conn
, virsh_conn
, processors
)
116 if not return_status
:
117 return (return_status
, 'Error at get_processor_information in '+machine
+': '+code
)
120 #Get the server memory information
121 memory_nodes
= dict()
122 (return_status
, code
) = get_memory_information(ssh_conn
, virsh_conn
, memory_nodes
)
123 if not return_status
:
124 return (return_status
, 'Error at get_memory_information in '+machine
+': '+code
)
127 #Get nics information
128 nic_topology
= dict()
129 # (return_status, code) = get_nic_information_old(ssh_conn, nic_topology)
130 (return_status
, code
) = get_nic_information(ssh_conn
, virsh_conn
, nic_topology
)
131 if not return_status
:
132 return (return_status
, 'Error at get_nic_information in '+machine
+': '+code
)
135 #Pack each processor, memory node and nics in a node element
136 #and add the node to the RAD element
137 for socket_id
, processor
in processors
.iteritems():
139 if not socket_id
in nic_topology
:
140 nic_topology
[socket_id
] = list()
142 (return_status
, code
) = node
.set(processor
, memory_nodes
[socket_id
], nic_topology
[socket_id
])
144 # (return_status, code) = node.set(processor, memory_nodes[socket_id])
145 if not return_status
:
146 return (return_status
, 'Error at node.set in '+machine
+': '+code
)
148 (return_status
, code
) = self
.insert_node(node
)
149 if not return_status
:
150 return (return_status
, 'Error at self.insert_node in '+machine
+': '+code
)
151 if code
not in warning_text
:
156 (return_status
, code
) = get_os_information(ssh_conn
, os
)
157 if not return_status
:
158 return (return_status
, 'Error at get_os_information in '+machine
+': '+code
)
160 (return_status
, code
) = self
.set_os(os
)
161 if not return_status
:
162 return (return_status
, 'Error at self.set_os in '+machine
+': '+code
)
165 #Fill hypervisor data
166 hypervisor
= Hypervisor()
167 (return_status
, code
) = get_hypervisor_information(virsh_conn
, hypervisor
)
168 if not return_status
:
169 return (return_status
, 'Error at get_hypervisor_information in '+machine
+': '+code
)
171 (return_status
, code
) = self
.set_hypervisor(hypervisor
)
172 if not return_status
:
173 return (return_status
, 'Error at self.set_hypervisor in '+machine
+': '+code
)
177 return (True, warning_text
)
178 except libvirt
.libvirtError
, e
:
179 text
= e
.get_error_message()
180 print 'RADclass.obtain_RAD() exception:', text
182 except paramiko
.ssh_exception
.SSHException
, e
:
184 print "obtain_RAD ssh Exception:", text
187 def set_name(self
,name
):
188 """Sets the machine name.
189 Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
190 if not isinstance(name
,str):
191 return (False, 'The variable \'name\' must be text')
195 def set_connection_info(self
, machine
, user
, password
):
196 """Sets the connection information.
197 Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
198 if not isinstance(machine
,str):
199 return (False, 'The variable \'machine\' must be text')
200 if not isinstance(user
,str):
201 return (False, 'The variable \'user\' must be text')
202 # if not isinstance(password,str):
203 # return (False, 'The variable \'password\' must be text')
204 (self
.machine
, self
.user
, self
.password
) = (machine
, user
, password
)
207 def insert_node(self
,node
):
208 """Inserts a new node and updates class variables.
209 Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
210 if not isinstance(node
,Node
):
211 return (False, 'The variable \'node\' must be a Node element')
213 if node
.id_
in self
.nodes
:
214 return (False, 'The node is already present in the nodes list.')
216 #Check if network ports have not been inserted previously as part of another node
217 for port_key
in node
.ports_list
:
218 if port_key
in self
.ports_list
:
219 return (False, 'Network port '+port_key
+' defined multiple times in the system')
220 self
.ports_list
.append(port_key
)
223 self
.nodes
[node
.id_
] = node
226 self
.update_variables()
230 def update_variables(self
):
231 """Updates class variables.
232 Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
234 #The number of processors and nodes is the same
235 self
.nr_processors
= len(self
.nodes
)
237 #If all processors are the same get the values. Otherwise keep them as none
238 prev_processor_family
= prev_processor_manufacturer
= prev_processor_version
= prev_processor_features
= None
239 different_processor_family
= different_processor_manufacturer
= different_processor_version
= different_processor_features
= False
240 for node
in self
.nodes
.itervalues():
241 (self
.processor_family
, self
.processor_manufacturer
, self
.processor_version
, self
.processor_features
) = node
.get_processor_info()
242 if prev_processor_family
!= None and self
.processor_family
!= prev_processor_family
:
243 different_processor_family
= True
244 if prev_processor_manufacturer
!= None and self
.processor_manufacturer
!= prev_processor_manufacturer
:
245 different_processor_manufacturer
= True
246 if prev_processor_version
!= None and self
.processor_version
!= prev_processor_version
:
247 different_processor_version
= True
248 if prev_processor_features
!= None and self
.processor_features
!= prev_processor_features
:
249 different_processor_features
= True
250 (prev_processor_family
, prev_processor_manufacturer
, prev_processor_version
, prev_processor_features
) = (self
.processor_family
, self
.processor_manufacturer
, self
.processor_version
, self
.processor_features
)
252 if different_processor_family
:
253 self
.processor_family
= None
254 if different_processor_features
:
255 self
.processor_features
= None
256 if different_processor_manufacturer
:
257 self
.processor_manufacturer
= None
258 if different_processor_version
:
259 self
.processor_version
= None
261 #If all memory nodes are the same get the values. Otherwise keep them as none
262 #Sum the total memory
264 different_memory_freq
= different_memory_nr_channels
= different_memory_type
= different_memory_hugepage_sz
= False
265 prev_memory_freq
= prev_memory_nr_channels
= prev_memory_type
= prev_memory_hugepage_sz
= None
266 for node
in self
.nodes
.itervalues():
267 (self
.memory_freq
, self
.memory_nr_channels
, self
.memory_type
, memory_size
, self
.memory_hugepage_sz
) = node
.get_memory_info()
268 self
.memory_size
+= memory_size
269 if prev_memory_freq
!= None and self
.memory_freq
!= prev_memory_freq
:
270 different_memory_freq
= True
271 if prev_memory_nr_channels
!= None and self
.memory_nr_channels
!= prev_memory_nr_channels
:
272 different_memory_nr_channels
= True
273 if prev_memory_type
!= None and self
.memory_type
!= prev_memory_type
:
274 different_memory_type
= True
275 if prev_memory_hugepage_sz
!= None and self
.memory_hugepage_sz
!= prev_memory_hugepage_sz
:
276 different_memory_hugepage_sz
= True
277 (prev_memory_freq
, prev_memory_nr_channels
, prev_memory_type
, prev_memory_hugepage_sz
) = (self
.memory_freq
, self
.memory_nr_channels
, self
.memory_type
, self
.memory_hugepage_sz
)
279 if different_memory_freq
:
280 self
.memory_freq
= None
281 if different_memory_nr_channels
:
282 self
.memory_nr_channels
= None
283 if different_memory_type
:
284 self
.memory_type
= None
285 if different_memory_hugepage_sz
:
286 warning_text
+= 'Detected different hugepages size in different sockets\n'
288 return (True, warning_text
)
290 def set_hypervisor(self
,hypervisor
):
291 """Sets the hypervisor.
292 Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
293 if not isinstance(hypervisor
,Hypervisor
):
294 return (False, 'The variable \'hypervisor\' must be of class Hypervisor')
296 self
.hypervisor
.assign(hypervisor
)
300 """Sets the operating system.
301 Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
302 if not isinstance(os
,OpSys
):
303 return (False, 'The variable \'os\' must be of class OpSys')
309 text
= 'name: '+str(self
.name
)+'\n'
310 text
+= 'processor:\n'
311 text
+= ' nr_processors: '+str(self
.nr_processors
)+'\n'
312 text
+= ' family: '+str(self
.processor_family
)+'\n'
313 text
+= ' manufacturer: '+str(self
.processor_manufacturer
)+'\n'
314 text
+= ' version: '+str(self
.processor_version
)+'\n'
315 text
+= ' features: '+str(self
.processor_features
)+'\n'
317 text
+= ' type: '+str(self
.memory_type
)+'\n'
318 text
+= ' freq: '+str(self
.memory_freq
)+'\n'
319 text
+= ' nr_channels: '+str(self
.memory_nr_channels
)+'\n'
320 text
+= ' size: '+str(self
.memory_size
)+'\n'
321 text
+= 'hypervisor:\n'
322 text
+= self
.hypervisor
.to_text()
324 text
+= self
.os
.to_text()
325 text
+= 'resource topology:\n'
326 text
+= ' nr_nodes: '+ str(len(self
.nodes
))+'\n'
328 for node_k
, node_v
in self
.nodes
.iteritems():
329 text
+= ' node'+str(node_k
)+':\n'
330 text
+= node_v
.to_text()
334 return yaml
.load(self
.to_text())
338 self
.id_
= None #Integer. Node id. Unique in the system
339 self
.processor
= ProcessorNode() #Information about the processor in the node
340 self
.memory
= MemoryNode() #Information about the memory in the node
341 self
.nic_list
= list() #List of Nic() containing information about the nics associated to the node
342 self
.ports_list
= list() #List containing all network ports in the node. This is used to avoid having defined multiple times the same port in the system
344 def get_processor_info(self
):
345 """Gets the processor information. Returns (processor_family, processor_manufacturer, processor_version, processor_features)"""
346 return self
.processor
.get_info()
348 def get_memory_info(self
):
349 """Gets the memory information. Returns (memory_freq, memory_nr_channels, memory_type, memory_size)"""
350 return self
.memory
.get_info()
352 # def set(self, *args):
353 # """Sets the node information. Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
355 # processor = args[0]
359 # processor = args[0]
364 # return (False, 'Wrong number of elements calling Node().set()')
366 def set(self
, processor
, memory
, nic_list
):
367 (status
, return_code
) = self
.processor
.assign(processor
)
369 return (status
, return_code
)
371 self
.id_
= processor
.id_
373 (status
, return_code
) = self
.memory
.assign(memory
)
375 return (status
, return_code
)
379 if not isinstance(nic
,Nic
):
380 return (False, 'The nics must be of type Nic')
381 self
.nic_list
.append(nic
)
382 for port_key
in nic
.ports
.iterkeys():
383 if port_key
in self
.ports_list
:
384 return (False, 'Network port '+port_key
+'defined multiple times in the same node')
385 self
.ports_list
.append(port_key
)
389 def assign(self
, node
):
390 """Sets the node information.
391 Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
393 processor
= node
.processor
395 nic_list
= node
.nic_list
396 (status
, return_code
) = self
.processor
.assign(processor
)
398 return (status
, return_code
)
400 self
.id_
= processor
.id_
402 (status
, return_code
) = self
.memory
.assign(memory
)
404 return (status
, return_code
)
408 if not isinstance(nic
,Nic
):
409 return (False, 'The nics must be of type Nic')
410 self
.nic_list
.append(nic
)
411 for port_key
in nic
.ports
.iterkeys():
412 if port_key
in self
.ports_list
:
413 return (False, 'Network port '+port_key
+'defined multiple times in the same node')
414 self
.ports_list
.append(port_key
)
416 return (True,warning_text
)
419 text
= ' id: '+str(self
.id_
)+'\n'
421 text
+= self
.processor
.to_text()
423 text
+= self
.memory
.to_text()
424 if len(self
.nic_list
) > 0:
427 for nic
in self
.nic_list
:
428 text
+= ' nic '+str(nic_index
)+':\n'
429 text
+= nic
.to_text()
433 class ProcessorNode():
434 #Definition of the possible values of processor variables
435 possible_features
= definitionsClass
.processor_possible_features
436 possible_manufacturers
= definitionsClass
.processor_possible_manufacturers
437 possible_families
= definitionsClass
.processor_possible_families
438 possible_versions
= definitionsClass
.processor_possible_versions
441 self
.id_
= None #Integer. Numeric identifier of the socket
442 self
.family
= None #Text. Family name of the processor
443 self
.manufacturer
= None #Text. Manufacturer of the processor
444 self
.version
= None #Text. Model version of the processor
445 self
.features
= list() #list. List of features offered by the processor
446 self
.cores
= list() #list. List of cores in the processor. In case of hyperthreading the coupled cores are expressed as [a,b]
447 self
.eligible_cores
= list()#list. List of cores that can be used
448 #self.decicated_cores
449 #self.shared_cores -> this should also contain information to know if cores are being used
451 def assign(self
, processor
):
452 """Sets the processor information.
453 Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
454 if not isinstance(processor
,ProcessorNode
):
455 return (False, 'The variable \'processor\' must be of class ProcessorNode')
457 self
.id_
= processor
.id_
458 self
.family
= processor
.family
459 self
.manufacturer
= processor
.manufacturer
460 self
.version
= processor
.version
461 self
.features
= processor
.features
462 self
.cores
= processor
.cores
463 self
.eligible_cores
= processor
.eligible_cores
467 def set(self
, id_
, family
, manufacturer
, version
, features
, cores
):
468 """Sets the processor information.
469 Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
472 if not isinstance(id_
,int):
473 return (False, 'The processor id_ must be of type int')
474 if not isinstance(family
,str):
475 return (False, 'The processor family must be of type str')
476 if not isinstance(manufacturer
,str):
477 return (False, 'The processor manufacturer must be of type str')
478 if not isinstance(version
,str):
479 return (False, 'The processor version must be of type str')
480 if not isinstance(features
,list):
481 return (False, 'The processor features must be of type list')
482 if not isinstance(cores
,list):
483 return (False, 'The processor cores must be of type list')
485 (self
.id_
, self
.family
, self
.manufacturer
, self
.version
) = (id_
, family
, manufacturer
, version
)
487 if not manufacturer
in self
.possible_manufacturers
:
488 warning_text
+= "processor manufacturer '%s' not among: %s\n" %(manufacturer
, str(self
.possible_manufacturers
))
489 if not family
in self
.possible_families
:
490 warning_text
+= "family '%s' not among: %s\n" % (family
, str(self
.possible_families
))
491 # if not version in self.possible_versions:
492 # warning_text += 'The version %s is not one of these: %s\n' % (version, str(self.possible_versions))
494 for feature
in features
:
495 if not feature
in self
.possible_features
:
496 warning_text
+= "processor feature '%s' not among: %s\n" % (feature
, str(self
.possible_versions
))
497 self
.features
.append(feature
)
499 for iterator
in sorted(cores
):
500 if not isinstance(iterator
,list) or not all(isinstance(x
, int) for x
in iterator
):
501 return (False, 'The cores list must be in the form of [[a,b],[c,d],...] where a,b,c,d are of type int')
502 self
.cores
.append(iterator
)
504 self
.set_eligible_cores()
506 return (True,warning_text
)
508 def set_eligible_cores(self
):
509 """Set the default eligible cores, this is all cores non used by the host operating system"""
511 for iterator
in self
.cores
:
513 self
.eligible_cores
.append(iterator
)
519 """Returns processor parameters (self.family, self.manufacturer, self.version, self.features)"""
520 return (self
.family
, self
.manufacturer
, self
.version
, self
.features
)
523 text
= ' id: '+str(self
.id_
)+'\n'
524 text
+= ' family: '+self
.family
+'\n'
525 text
+= ' manufacturer: '+self
.manufacturer
+'\n'
526 text
+= ' version: '+self
.version
+'\n'
527 text
+= ' features: '+str(self
.features
)+'\n'
528 text
+= ' cores: '+str(self
.cores
)+'\n'
529 text
+= ' eligible_cores: '+str(self
.eligible_cores
)+'\n'
534 self
.modules
= list() #List of MemoryModule(). List of all modules installed in the node
535 self
.nr_channels
= None #Integer. Number of modules installed in the node
536 self
.node_size
= None #Integer. Total size in KiB of memory installed in the node
537 self
.eligible_memory
= None #Integer. Size in KiB of eligible memory in the node
538 self
.hugepage_sz
= None #Integer. Size in KiB of hugepages
539 self
.hugepage_nr
= None #Integer. Number of hugepages allocated in the module
540 self
.eligible_hugepage_nr
= None #Integer. Number of eligible hugepages in the node
541 self
.type_
= None #Text. Type of memory modules. If modules have a different value keep it as None
542 self
.freq
= None #Integer. Frequency of the modules in MHz. If modules have a different value keep it as None
543 self
.module_size
= None #Integer. Size of the modules in KiB. If modules have a different value keep it as None
544 self
.form_factor
= None #Text. Form factor of the modules. If modules have a different value keep it as None
546 def assign(self
, memory_node
):
547 return self
.set(memory_node
.modules
, memory_node
.hugepage_sz
, memory_node
.hugepage_nr
)
549 def set(self
, modules
, hugepage_sz
, hugepage_nr
):
550 """Set the memory node information. hugepage_sz must be expressed in KiB.
551 Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
552 if not isinstance(modules
, list):
553 return (False, 'The modules must be a list of elements of class MemoryModule')
554 if not isinstance(hugepage_sz
,int):
555 return (False, 'The hugepage_sz variable must be an int expressing the size in KiB')
556 if not isinstance(hugepage_nr
,int):
557 return (False, 'The hugepage_nr variable must be of type int')
559 (self
.hugepage_sz
, self
.hugepage_nr
) = (hugepage_sz
, hugepage_nr
)
560 self
.node_size
= self
.nr_channels
= 0
562 different_type
= different_freq
= different_module_size
= different_form_factor
= False
563 prev_type
= prev_freq
= prev_module_size
= prev_form_factor
= None
564 for iterator
in modules
:
565 if not isinstance(iterator
,MemoryModule
):
566 return (False, 'The modules must be a list of elements of class MemoryModule')
567 self
.modules
.append(iterator
)
568 (self
.type_
, self
.freq
, self
.module_size
, self
.form_factor
) = (iterator
.type_
, iterator
.freq
, iterator
.size
, iterator
.form_factor
)
569 self
.node_size
+= self
.module_size
570 self
.nr_channels
+= 1
571 if prev_type
!= None and prev_type
!= self
.type_
:
572 different_type
= True
573 if prev_freq
!= None and prev_freq
!= self
.freq
:
574 different_freq
= True
575 if prev_module_size
!= None and prev_module_size
!= self
.module_size
:
576 different_module_size
= True
577 if prev_form_factor
!= None and prev_form_factor
!= self
.form_factor
:
578 different_form_factor
= True
579 (prev_type
, prev_freq
, prev_module_size
, prev_form_factor
) = (self
.type_
, self
.freq
, self
.module_size
, self
.form_factor
)
585 if different_module_size
:
586 self
.module_size
= None
587 if different_form_factor
:
588 self
.form_factor
= None
590 (return_value
, error_code
) = self
.set_eligible_memory()
592 return (return_value
, error_code
)
596 def set_eligible_memory(self
):
597 """Sets the default eligible_memory and eligible_hugepage_nr. This is all memory but 2GiB and all hugepages"""
598 self
.eligible_memory
= self
.node_size
- 2*1024*1024
599 if self
.eligible_memory
< 0:
600 return (False, "There is less than 2GiB of memory in the module")
602 self
.eligible_hugepage_nr
= self
.hugepage_nr
606 """Return memory information (self.freq, self.nr_channels, self.type_, self.node_size)"""
607 return (self
.freq
, self
.nr_channels
, self
.type_
, self
.node_size
, self
.hugepage_sz
)
610 text
= ' node_size: '+str(self
.node_size
)+'\n'
611 text
+= ' nr_channels: '+str(self
.nr_channels
)+'\n'
612 text
+= ' eligible_memory: '+str(self
.eligible_memory
)+'\n'
613 text
+= ' hugepage_sz: '+str(self
.hugepage_sz
)+'\n'
614 text
+= ' hugepage_nr: '+str(self
.hugepage_nr
)+'\n'
615 text
+= ' eligible_hugepage_nr: '+str(self
.eligible_hugepage_nr
)+'\n'
616 text
+= ' type: '+self
.type_
+'\n'
617 text
+= ' freq: '+str(self
.freq
)+'\n'
618 text
+= ' module_size: '+str(self
.module_size
)+'\n'
619 text
+= ' form_factor: '+self
.form_factor
+'\n'
620 text
+= ' modules details:\n'
621 for module
in self
.modules
:
622 text
+= module
.to_text()
625 class MemoryModule():
626 #Definition of the possible values of module variables
627 possible_types
= definitionsClass
.memory_possible_types
628 possible_form_factors
= definitionsClass
.memory_possible_form_factors
631 self
.locator
= None #Text. Name of the memory module
632 self
.type_
= None #Text. Type of memory module
633 self
.freq
= None #Integer. Frequency of the module in MHz
634 self
.size
= None #Integer. Size of the module in KiB
635 self
.form_factor
= None #Text. Form factor of the module
637 def set(self
, locator
, type_
, freq
, size
, form_factor
):
638 """Sets the memory module information.
639 Frequency must be expressed in MHz and size in KiB.
640 Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
642 if not isinstance(locator
, str):
643 return (False, "The type of the variable locator must be str")
644 if not isinstance(type_
, str):
645 return (False, "The type of the variable type_ must be str")
646 if not isinstance(form_factor
, str):
647 return (False, "The type of the variable form_factor must be str")
648 if not isinstance(freq
, int):
649 return (False, "The type of the variable freq must be int")
650 if not isinstance(size
, int):
651 return (False, "The type of the variable size must be int")
653 if not form_factor
in self
.possible_form_factors
:
654 warning_text
+= "memory form_factor '%s' not among: %s\n" %(form_factor
, str(self
.possible_form_factors
))
655 if not type_
in self
.possible_types
:
656 warning_text
+= "memory type '%s' not among: %s\n" %(type_
, str(self
.possible_types
))
658 (self
.locator
, self
.type_
, self
.freq
, self
.size
, self
.form_factor
) = (locator
, type_
, freq
, size
, form_factor
)
659 return (True, warning_text
)
662 text
= ' '+self
.locator
+':\n'
663 text
+= ' type: '+self
.type_
+'\n'
664 text
+= ' freq: '+str(self
.freq
)+'\n'
665 text
+= ' size: '+str(self
.size
)+'\n'
666 text
+= ' form factor: '+self
.form_factor
+'\n'
671 self
.model
= None #Text. Model of the nic
672 self
.ports
= dict() #Dictionary of ports. Keys are the port name, value are Port() elements
674 def set_model(self
, model
):
675 """Sets the model of the nic. Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
676 if not isinstance(model
,str):
677 return (False, 'The \'model\' must be of type str')
682 def add_port(self
, port
):
683 """Adds a port to the nic. Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
684 if not isinstance(port
,Port
):
685 return (False, 'The \'port\' must be of class Port')
687 # port_id = str(port.pci_device_id[0])+':'+str(port.pci_device_id[1])+':'+str(port.pci_device_id[2])+'.'+str(port.pci_device_id[3])
689 # port_id = port.name
690 port_id
= port
.pci_device_id
692 if port_id
in self
.ports
:
693 return (False, 'The \'port\' '+port
.pci_device_id
+' is duplicated in the nic')
694 # return (False, 'The \'port\' is duplicated in the nic')
696 self
.ports
[port_id
] = port
700 text
= ' model: '+ str(self
.model
)+'\n'
701 text
+= ' ports: '+'\n'
702 for key
,port
in self
.ports
.iteritems():
703 text
+= ' "'+key
+'":'+'\n'
704 text
+= port
.to_text()
709 self
.name
= None #Text. Port name
710 self
.virtual
= None #Boolean. States if the port is a virtual function
711 self
.enabled
= None #Boolean. States if the port is enabled
712 self
.eligible
= None #Boolean. States if the port is eligible
713 self
.speed
= None #Integer. Indicates the speed in Mbps
714 self
.available_bw
= None #Integer. BW in Mbps that is available.
715 self
.mac
= None #list. Indicates the mac address of the port as a list in format ['XX','XX','XX','XX','XX','XX']
716 self
.pci_device_id_split
= None #list. Indicates the pci address of the port as a list in format ['XXXX','XX','XX','X']
717 self
.pci_device_id
= None
718 self
.PF_pci_device_id
= None
720 # def set(self, name, virtual, enabled, speed, mac, pci_device_id, pci_device_id_split):
721 # """Sets the port information. The variable speed indicates the speed in Mbps. Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
722 # if not isinstance(name,str):
723 # return (False, 'The variable \'name\' must be of type str')
724 # if not isinstance(virtual,bool):
725 # return (False, 'The variable \'virtual\' must be of type bool')
726 # if not isinstance(enabled,bool):
727 # return (False, 'The variable \'enabled\' must be of type bool')
728 # if not isinstance(enabled,bool):
729 # return (speed, 'The variable \'speed\' must be of type int')
730 # if not isinstance(mac, list) and not isinstance(mac,NoneType):
731 # return (False, 'The variable \'enabled\' must be of type list indicating the mac address in format [\'XXXX\',\'XX\',\'XX\',\'X\'] or NoneType')
732 # if not isinstance(pci_device_id_split, list) or len(pci_device_id_split) != 4:
733 # return (False, 'The variable \'pci_device_id_split\' must be of type list, indicating the pci address in format [\'XX\',\'XX\',\'XX\',\'XX\',\'XX\',\'XX\']')
735 # expected_len = [4,2,2,1]
737 # for iterator in pci_device_id_split:
738 # if not isinstance(iterator,str) or not iterator.isdigit() or len(iterator) != expected_len[index]:
739 # return (False, 'The variable \'pci_device_id_split\' must be of type list, indicating the pci address in format [\'XX\',\'XX\',\'XX\',\'XX\',\'XX\',\'XX\']')
742 # if not isinstance(mac,NoneType):
743 # for iterator in mac:
744 # if not isinstance(iterator,str) or not iterator.isalnum() or len(iterator) != 2:
745 # return (False, 'The variable \'enabled\' must be of type list indicating the mac address in format [\'XXXX\',\'XX\',\'XX\',\'X\'] or NoneType')
747 # #By default only virtual ports are eligible
748 # # (self.name, self.virtual, self.enabled, self.eligible, self.available_bw, self.speed, self.mac, self.pci_device_id, self.pci_device_id_split) = (name, virtual, enabled, virtual, speed, speed, mac, pci_device_id, pci_device_id_split)
749 # (self.name, self.virtual, self.enabled, self.eligible, self.available_bw, self.mac, self.pci_device_id, self.pci_device_id_split) = (name, virtual, enabled, virtual, speed, mac, pci_device_id, pci_device_id_split)
752 text
= ' pci: "'+ str(self
.pci_device_id
)+'"\n'
753 text
+= ' virtual: '+ str(self
.virtual
)+'\n'
755 text
+= ' PF_pci_id: "'+self
.PF_pci_device_id
+'"\n'
756 text
+= ' eligible: '+ str(self
.eligible
)+'\n'
757 text
+= ' enabled: '+str(self
.enabled
)+'\n'
758 text
+= ' speed: '+ str(self
.speed
)+'\n'
759 text
+= ' available bw: '+ str(self
.available_bw
)+'\n'
760 text
+= ' mac: '+ str(self
.mac
)+'\n'
761 text
+= ' source_name: '+ str(self
.name
)+'\n'
765 #Definition of the possible values of hypervisor variables
766 possible_types
= definitionsClass
.hypervisor_possible_types
767 possible_domain_types
= definitionsClass
.hypervisor_possible_domain_types
770 self
.type_
= None #Text. Hypervisor type_
771 self
.version
= None #int. Hypervisor version
772 self
.lib_version
= None #int. Libvirt version used to compile hypervisor
773 self
.domains
= list() #list. List of all the available domains
775 def set(self
, hypervisor
, version
, lib_version
, domains
):
777 if not isinstance(hypervisor
,str):
778 return (False, 'The variable type_ must be of type str')
779 if not isinstance(version
,int):
780 return (False, 'The variable version must be of type int')
781 if not isinstance(lib_version
,int):
782 return (False, 'The library version must be of type int')
783 if not isinstance(domains
,list):
784 return (False, 'Domains must be a list of the possible domains as str')
786 if not hypervisor
in self
.possible_types
:
787 warning_text
+= "Hyperpivor '%s' not among: %s\n" % (hypervisor
, str(self
.possible_types
))
789 valid_domain_found
= False
790 for domain
in domains
:
791 if not isinstance(domain
,str):
792 return (False, 'Domains must be a list of the possible domains as str')
793 if domain
in self
.possible_domain_types
:
794 valid_domain_found
= True
795 self
.domains
.append(domain
)
797 if not valid_domain_found
:
798 warning_text
+= 'No valid domain found among: %s\n' % str(self
.possible_domain_types
)
801 (self
.version
, self
.lib_version
, self
.type_
) = (version
, lib_version
, hypervisor
)
802 return (True, warning_text
)
804 def assign(self
, hypervisor
):
805 (self
.version
, self
.lib_version
, self
.type_
) = (hypervisor
.version
, hypervisor
.lib_version
, hypervisor
.type_
)
806 for domain
in hypervisor
.domains
:
807 self
.domains
.append(domain
)
811 text
= ' type: '+self
.type_
+'\n'
812 text
+= ' version: '+str(self
.version
)+'\n'
813 text
+= ' libvirt version: '+ str(self
.lib_version
)+'\n'
814 text
+= ' domains: '+str(self
.domains
)+'\n'
818 #Definition of the possible values of os variables
819 possible_id
= definitionsClass
.os_possible_id
820 possible_types
= definitionsClass
.os_possible_types
821 possible_architectures
= definitionsClass
.os_possible_architectures
824 self
.id_
= None #Text. Identifier of the OS. Formed by <Distibutor ID>-<Release>-<Codename>. In linux this can be obtained using lsb_release -a
825 self
.type_
= None #Text. Type of operating system
826 self
.bit_architecture
= None #Integer. Architecture
828 def set(self
, id_
, type_
, bit_architecture
):
830 if not isinstance(type_
,str):
831 return (False, 'The variable type_ must be of type str')
832 if not isinstance(id_
,str):
833 return (False, 'The variable id_ must be of type str')
834 if not isinstance(bit_architecture
,str):
835 return (False, 'The variable bit_architecture must be of type str')
837 if not type_
in self
.possible_types
:
838 warning_text
+= "os type '%s' not among: %s\n" %(type_
, str(self
.possible_types
))
839 if not id_
in self
.possible_id
:
840 warning_text
+= "os release '%s' not among: %s\n" %(id_
, str(self
.possible_id
))
841 if not bit_architecture
in self
.possible_architectures
:
842 warning_text
+= "os bit_architecture '%s' not among: %s\n" % (bit_architecture
, str(self
.possible_architectures
))
844 (self
.id_
, self
.type_
, self
.bit_architecture
) = (id_
, type_
, bit_architecture
)
845 return (True, warning_text
)
848 (self
.id_
, self
.type_
, self
.bit_architecture
) = (os
.id_
, os
.type_
, os
.bit_architecture
)
852 text
= ' id: '+self
.id_
+'\n'
853 text
+= ' type: '+self
.type_
+'\n'
854 text
+= ' bit_architecture: '+self
.bit_architecture
+'\n'
857 def get_hostname(virsh_conn
):
858 return virsh_conn
.getHostname().rstrip('\n')
860 def get_hugepage_size(ssh_conn
):
861 command
= 'sudo hugeadm --page-sizes'
862 # command = 'hugeadm --page-sizes-all'
863 (_
, stdout
, stderr
) = ssh_conn
.exec_command(command
)
864 error
= stderr
.read()
866 raise paramiko
.ssh_exception
.SSHException(command
+' : '+ error
)
872 def get_hugepage_nr(ssh_conn
,hugepage_sz
, node_id
):
873 command
= 'cat /sys/devices/system/node/node'+str(node_id
)+'/hugepages/hugepages-'+str(hugepage_sz
/1024)+'kB/nr_hugepages'
874 (_
, stdout
, _
) = ssh_conn
.exec_command(command
)
876 #text = stdout.read()
881 value
=int(stdout
.read())
886 def get_memory_information(ssh_conn
, virsh_conn
, memory_nodes
):
888 tree
=ElementTree
.fromstring(virsh_conn
.getSysinfo(0))
890 node_id
= 0 #TODO revise. Added for allowing VM as compute hosts
891 for target
in tree
.findall("memory_device"):
892 locator_f
= size_f
= freq_f
= type_f
= formfactor_f
= False
893 locator_f
= True #TODO revise. Added for allowing VM as compute hosts
894 module_form_factor
= ""
895 for entry
in target
.findall("entry"):
896 if entry
.get("name") == 'size':
898 size_split
= entry
.text
.split(' ')
899 if size_split
[1] == 'MB':
900 module_size
= int(size_split
[0]) * 1024 * 1024
901 elif size_split
[1] == 'GB':
902 module_size
= int(size_split
[0]) * 1024 * 1024 * 1024
903 elif size_split
[1] == 'KB':
904 module_size
= int(size_split
[0]) * 1024
906 module_size
= int(size_split
[0])
908 elif entry
.get("name") == 'speed':
910 freq_split
= entry
.text
.split(' ')
911 if freq_split
[1] == 'MHz':
912 module_freq
= int(freq_split
[0]) * 1024 * 1024
913 elif freq_split
[1] == 'GHz':
914 module_freq
= int(freq_split
[0]) * 1024 * 1024 * 1024
915 elif freq_split
[1] == 'KHz':
916 module_freq
= int(freq_split
[0]) * 1024
918 elif entry
.get("name") == 'type':
920 module_type
= entry
.text
922 elif entry
.get("name") == 'form_factor':
924 module_form_factor
= entry
.text
925 #TODO revise. Commented for allowing VM as compute hosts
926 # elif entry.get("name") == 'locator' and not locator_f:
927 # # other case, it is obtained by bank_locator that we give priority to
928 # locator = entry.text
929 # pos = locator.find(module_form_factor)
930 # if module_form_factor == locator[0:len(module_form_factor) ]:
931 # pos = len(module_form_factor) +1
934 # if locator[pos] in "ABCDEFGH":
936 # node_id = ord(locator[pos])-ord('A')
937 # #print entry.text, node_id
939 # elif entry.get("name") == 'bank_locator':
940 # locator = entry.text
941 # pos = locator.find("NODE ")
942 # if pos >= 0 and len(locator)>pos+5:
943 # if locator[pos+5] in ("01234567"): #len("NODE ") is 5
944 # node_id = int(locator[pos+5])
948 #When all module fields have been found add a new module to the list
949 if locator_f
and size_f
and freq_f
and type_f
and formfactor_f
:
950 #If the memory node has not yet been created create it
951 if node_id
not in memory_dict
:
952 memory_dict
[node_id
] = []
954 #Add a new module to the memory node
955 module
= MemoryModule()
956 #TODO revise. Changed for allowing VM as compute hosts
957 (return_status
, code
) = module
.set('NODE %d' % node_id
, module_type
, module_freq
, module_size
, module_form_factor
)
958 #(return_status, code) = module.set(locator, module_type, module_freq, module_size, module_form_factor)
959 if not return_status
:
960 return (return_status
, code
)
961 memory_dict
[node_id
].append(module
)
962 if code
not in warning_text
:
964 node_id
+= 1 #TODO revise. Added for allowing VM as compute hosts
967 #Hugepage size is constant for all nodes
968 hugepage_sz
= get_hugepage_size(ssh_conn
)
969 for node_id
, modules
in memory_dict
.iteritems():
970 memory_node
= MemoryNode()
971 memory_node
.set(modules
, hugepage_sz
, get_hugepage_nr(ssh_conn
,hugepage_sz
, node_id
))
972 memory_nodes
[node_id
] = memory_node
974 return (True, warning_text
)
976 def get_cpu_topology_ht(ssh_conn
, topology
):
977 command
= 'cat /proc/cpuinfo'
978 (_
, stdout
, stderr
) = ssh_conn
.exec_command(command
)
979 error
= stderr
.read()
981 raise paramiko
.ssh_exception
.SSHException(command
+' : '+ error
)
987 for line
in stdout
.readlines():
988 if len(line
.strip()) != 0:
989 name
, value
= line
.split(":", 1)
990 core_lines
[name
.strip()] = value
.strip()
992 core_details
.append(core_lines
)
995 for core
in core_details
:
996 for field
in ["processor", "core id", "physical id"]:
997 if field
not in core
:
998 return(False,'Error getting '+field
+' value from /proc/cpuinfo')
999 core
[field
] = int(core
[field
])
1001 if core
["core id"] not in cores
:
1002 cores
.append(core
["core id"])
1003 if core
["physical id"] not in sockets
:
1004 sockets
.append(core
["physical id"])
1005 key
= (core
["physical id"], core
["core id"])
1006 if key
not in core_map
:
1008 core_map
[key
].append(core
["processor"])
1011 hyperthreaded_cores
= list()
1013 hyperthreaded_cores
.append(core_map
[(s
,c
)])
1014 topology
[s
] = hyperthreaded_cores
1018 def get_processor_information(ssh_conn
, vish_conn
, processors
):
1020 #Processor features are the same for all processors
1021 #TODO (at least using virsh capabilities)nr_numa_nodes
1022 capabilities
= list()
1023 tree
=ElementTree
.fromstring(vish_conn
.getCapabilities())
1024 for target
in tree
.findall("host/cpu/feature"):
1025 if target
.get("name") == 'pdpe1gb':
1026 capabilities
.append('lps')
1027 elif target
.get("name") == 'dca':
1028 capabilities
.append('dioc')
1029 elif target
.get("name") == 'vmx' or target
.get("name") == 'svm':
1030 capabilities
.append('hwsv')
1031 elif target
.get("name") == 'ht':
1032 capabilities
.append('ht')
1034 target
= tree
.find("host/cpu/arch")
1035 if target
.text
== 'x86_64' or target
.text
== 'amd64':
1036 capabilities
.append('64b')
1038 command
= 'cat /proc/cpuinfo | grep flags'
1039 (_
, stdout
, stderr
) = ssh_conn
.exec_command(command
)
1040 error
= stderr
.read()
1042 raise paramiko
.ssh_exception
.SSHException(command
+' : '+ error
)
1043 line
= stdout
.readline()
1044 if 'ept' in line
or 'npt' in line
:
1045 capabilities
.append('tlbps')
1047 #Find out if IOMMU is enabled
1048 command
= 'dmesg |grep -e Intel-IOMMU'
1049 (_
, stdout
, stderr
) = ssh_conn
.exec_command(command
)
1050 error
= stderr
.read()
1052 raise paramiko
.ssh_exception
.SSHException(command
+' : '+ error
)
1053 if 'enabled' in stdout
.read():
1054 capabilities
.append('iommu')
1057 command
= 'dmesg |grep -e AMD-Vi'
1058 (_
, stdout
, stderr
) = ssh_conn
.exec_command(command
)
1059 error
= stderr
.read()
1061 raise paramiko
.ssh_exception
.SSHException(command
+' : '+ error
)
1062 if len(stdout
.read()) > 0:
1063 capabilities
.append('iommu')
1065 #-----------------------------------------------------------
1067 #In case hyperthreading is active it is necessary to determine cpu topology using /proc/cpuinfo
1068 if 'ht' in capabilities
:
1069 (return_status
, code
) = get_cpu_topology_ht(ssh_conn
, topology
)
1070 if not return_status
:
1071 return (return_status
, code
)
1072 warning_text
+= code
1074 #Otherwise it is possible to do it using virsh capabilities
1076 for target
in tree
.findall("host/topology/cells/cell"):
1077 socket_id
= int(target
.get("id"))
1078 topology
[socket_id
] = list()
1079 for cpu
in target
.findall("cpus/cpu"):
1080 topology
[socket_id
].append(int(cpu
.get("id")))
1082 #-----------------------------------------------------------
1083 #Create a dictionary with the information of all processors
1084 #p_fam = p_man = p_ver = None
1085 tree
=ElementTree
.fromstring(vish_conn
.getSysinfo(0))
1086 #print vish_conn.getSysinfo(0)
1087 #return (False, 'forces error for debuging')
1089 socket_id
= -1 #in case we can not determine the socket_id we assume incremental order, starting by 0
1090 for target
in tree
.findall("processor"):
1093 #Get processor id, family, manufacturer and version
1094 for entry
in target
.findall("entry"):
1095 if entry
.get("name") == "status":
1096 if entry
.text
[0:11] == "Unpopulated":
1098 elif entry
.get("name") == 'socket_destination':
1099 socket_text
= entry
.text
1100 if socket_text
.startswith('CPU'):
1101 socket_text
= socket_text
.strip('CPU')
1102 socket_text
= socket_text
.strip() #removes trailing spaces
1103 if socket_text
.isdigit() and int(socket_text
)<9 and int(socket_text
)>0:
1104 socket_id
= int(socket_text
) - 1
1106 elif entry
.get("name") == 'family':
1109 elif entry
.get("name") == 'manufacturer':
1110 manufacturer
= entry
.text
1112 elif entry
.get("name") == 'version':
1113 version
= entry
.text
.strip()
1116 return (False, 'Error. Not all expected fields could be found in processor')
1118 #Create and fill processor structure
1120 continue #avoid inconsistence of some machines where more socket detected than
1121 processor
= ProcessorNode()
1122 (return_status
, code
) = processor
.set(socket_id
, family
, manufacturer
, version
, capabilities
, topology
[socket_id
])
1123 if not return_status
:
1124 return (return_status
, code
)
1125 if code
not in warning_text
:
1126 warning_text
+= code
1128 #Add processor to the processors dictionary
1129 processors
[socket_id
] = processor
1131 return (True, warning_text
)
1133 def get_nic_information(ssh_conn
, virsh_conn
, nic_topology
):
1135 #Get list of net devices
1136 net_devices
= virsh_conn
.listDevices('net',0)
1137 print virsh_conn
.listDevices('net',0)
1138 for device
in net_devices
:
1140 #Get the XML descriptor of the device:
1141 net_XML
= ElementTree
.fromstring(virsh_conn
.nodeDeviceLookupByName(device
).XMLDesc(0))
1142 #print "net_XML:" , net_XML
1144 parent
= net_XML
.find('parent')
1146 print 'No parent was found in XML for device '+device
1147 #Error. continue?-------------------------------------------------------------
1149 if parent
.text
== 'computer':
1151 if not parent
.text
.startswith('pci_'):
1152 print device
+ ' parent is neither computer nor pci'
1153 #Error. continue?-------------------------------------------------------------
1155 interface
= net_XML
.find('capability/interface').text
1156 mac
= net_XML
.find('capability/address').text
1159 pci_XML
= ElementTree
.fromstring(virsh_conn
.nodeDeviceLookupByName(parent
.text
).XMLDesc(0))
1162 name
= pci_XML
.find('name').text
.split('_')
1163 pci
= name
[1]+':'+name
[2]+':'+name
[3]+'.'+name
[4]
1165 #If slot == 0 it is a PF, otherwise it is a VF
1166 capability
= pci_XML
.find('capability')
1167 if capability
.get('type') != 'pci':
1168 print device
+ 'Capability is not of type pci in '+parent
.text
1169 #Error. continue?-------------------------------------------------------------
1171 slot
= capability
.find('slot').text
1172 bus
= capability
.find('bus').text
1174 numa_
= capability
.find('numa')
1176 node_id
= numa_
.get('node');
1177 if node_id
!= None: node_id
=int(node_id
)
1178 if slot
== None or bus
== None:
1179 print device
+ 'Bus and slot not detected in '+parent
.text
1180 #Error. continue?-------------------------------------------------------------
1183 # print ElementTree.tostring(pci_XML)
1185 capability_pf
= capability
.find('capability')
1186 if capability_pf
.get('type') != 'phys_function':
1187 print 'physical_function not found in VF '+parent
.text
1188 #Error. continue?-------------------------------------------------------------
1190 PF_pci
= capability_pf
.find('address').attrib
1191 PF_pci_text
= PF_pci
['domain'].split('x')[1]+':'+PF_pci
['bus'].split('x')[1]+':'+PF_pci
['slot'].split('x')[1]+'.'+PF_pci
['function'].split('x')[1]
1196 #Obtain node for the port
1198 node_id
= int(bus
)>>6
1199 #print "node_id:", node_id
1201 #Only for non virtual interfaces: Obtain speed and if link is detected (this must be done using ethtool)
1203 command
= 'sudo ethtool '+interface
+' | grep -e Speed -e "Link detected"'
1204 (_
, stdout
, stderr
) = ssh_conn
.exec_command(command
)
1205 error
= stderr
.read()
1207 print 'Error running '+command
+'\n'+error
1208 #Error. continue?-------------------------------------------------------------
1210 for line
in stdout
.readlines():
1211 line
= line
.strip().rstrip('\n').split(': ')
1212 if line
[0] == 'Speed':
1213 if line
[1].endswith('Mb/s'):
1214 speed
= int(line
[1].split('M')[0])*int(1e6
)
1215 elif line
[1].endswith('Gb/s'):
1216 speed
= int(line
[1].split('G')[0])*int(1e9
)
1217 elif line
[1].endswith('Kb/s'):
1218 speed
= int(line
[1].split('K')[0])*int(1e3
)
1220 #the interface is listed but won't be used
1222 elif line
[0] == 'Link detected':
1223 if line
[1] == 'yes':
1228 print 'Unnexpected output of command '+command
+':'
1230 #Error. continue?-------------------------------------------------------------
1233 if not node_id
in nic_topology
:
1234 nic_topology
[node_id
] = list()
1235 #With this implementation we make the RAD with only one nic per node and this nic has all ports, TODO: change this by including parent information of PF
1236 nic_topology
[node_id
].append(Nic())
1238 #Load the appropriate nic
1239 nic
= nic_topology
[node_id
][0]
1241 #Create a new port and fill it
1243 port
.name
= interface
1244 port
.virtual
= virtual
1247 port
.available_bw
= 0
1248 port
.PF_pci_device_id
= PF_pci_text
1250 port
.available_bw
= speed
1252 port
.enabled
= False
1254 port
.enabled
= enabled
1256 port
.eligible
= virtual
#Only virtual ports are eligible
1258 port
.pci_device_id
= pci
1259 port
.pci_device_id_split
= name
[1:]
1261 #Save the port information
1264 print 'Error: '+str(e
)
1266 #set in vitual ports if they are enabled
1267 for nic
in nic_topology
.itervalues():
1268 for port
in nic
[0].ports
.itervalues():
1269 # print port.pci_device_id
1271 enabled
= nic
[0].ports
.get(port
.PF_pci_device_id
)
1273 return(False, 'The PF '+port
.PF_pci_device_id
+' (VF '+port
.pci_device_id
+') is not present in ports dict')
1274 #Only if the PF is enabled the VF can be enabled
1275 if nic
[0].ports
[port
.PF_pci_device_id
].enabled
:
1278 port
.enabled
= False
1280 return (True, warning_text
)
1282 def get_nic_information_old(ssh_conn
, nic_topology
):
1283 command
= 'lstopo-no-graphics --of xml'
1284 (_
, stdout
, stderr
) = ssh_conn
.exec_command(command
)
1285 error
= stderr
.read()
1287 raise paramiko
.ssh_exception
.SSHException(command
+' : '+ error
)
1288 tree
=ElementTree
.fromstring(stdout
.read())
1289 for target
in tree
.findall("object/object"):
1291 if target
.get("type") != "NUMANode":
1293 node_id
= int(target
.get("os_index"))
1294 nic_topology
[node_id
] = list()
1296 #find nics in numa node
1297 for entry
in target
.findall("object/object"):
1298 if entry
.get("type") != 'Bridge':
1300 nic_name
= entry
.get("name")
1305 for pcidev
in entry
.findall("object"):
1306 if pcidev
.get("type") != 'PCIDev':
1308 enabled
= speed
= mac
= pci_busid
= None
1310 model
= pcidev
.get("name")
1312 if 'Virtual' in model
:
1314 pci_busid
= pcidev
.get("pci_busid")
1315 for osdev
in pcidev
.findall("object"):
1316 name
= osdev
.get("name")
1317 for info
in osdev
.findall("info"):
1318 if info
.get("name") != 'Address':
1320 mac
= info
.get("value")
1321 #get the port speed and status
1322 command
= 'sudo ethtool '+name
1323 (_
, stdout
, stderr
) = ssh_conn
.exec_command(command
)
1324 error
= stderr
.read()
1326 return (False, 'Error obtaining '+name
+' information: '+error
)
1327 ethtool
= stdout
.read()
1328 if '10000baseT/Full' in ethtool
:
1330 elif '1000baseT/Full' in ethtool
:
1332 elif '100baseT/Full' in ethtool
:
1334 elif '10baseT/Full' in ethtool
:
1337 return (False, 'Speed not detected in '+name
)
1340 if 'Link detected: yes' in ethtool
:
1343 if speed
!= None and mac
!= None and pci_busid
!= None:
1344 mac
= mac
.split(':')
1345 pci_busid_split
= re
.split(':|\.', pci_busid
)
1346 #Fill the port information
1347 port
.set(name
, virtual
, enabled
, speed
, mac
, pci_busid
, pci_busid_split
)
1350 if len(nic
.ports
) > 0:
1353 nic
.set_model(model
)
1355 nic
.set_model(nic_name
)
1357 #Add it to the topology
1358 nic_topology
[node_id
].append(nic
)
1362 def get_os_information(ssh_conn
, os
):
1364 # command = 'lsb_release -a'
1365 # (stdin, stdout, stderr) = ssh_conn.exec_command(command)
1367 # for line in stdout.readlines():
1368 # line_split = re.split('\t| *', line.rstrip('\n'))
1369 # if line_split[0] == 'Distributor' and line_split[1] == 'ID:':
1370 # distributor = line_split[2]
1372 # elif line_split[0] == 'Release:':
1373 # release = line_split[1]
1375 # elif line_split[0] == 'Codename:':
1376 # codename = line_split[1]
1379 # return (False, 'It was not possible to obtain the OS id')
1380 # id_ = distributor+'-'+release+'-'+codename
1383 command
= 'cat /etc/redhat-release'
1384 (_
, stdout
, _
) = ssh_conn
.exec_command(command
)
1385 id_text
= stdout
.read()
1388 command
= 'lsb_release -d -s'
1389 (_
, stdout
, _
) = ssh_conn
.exec_command(command
)
1390 id_text
= stdout
.read()
1392 raise paramiko
.ssh_exception
.SSHException("Can not determinte release neither with 'lsb_release' nor with 'cat /etc/redhat-release'")
1393 id_
= id_text
.rstrip('\n')
1395 command
= 'uname -o'
1396 (_
, stdout
, stderr
) = ssh_conn
.exec_command(command
)
1397 error
= stderr
.read()
1399 raise paramiko
.ssh_exception
.SSHException(command
+' : '+ error
)
1400 type_
= stdout
.read().rstrip('\n')
1402 command
= 'uname -i'
1403 (_
, stdout
, stderr
) = ssh_conn
.exec_command(command
)
1404 error
= stderr
.read()
1406 raise paramiko
.ssh_exception
.SSHException(command
+' : '+ error
)
1407 bit_architecture
= stdout
.read().rstrip('\n')
1409 (return_status
, code
) = os
.set(id_
, type_
, bit_architecture
)
1410 if not return_status
:
1411 return (return_status
, code
)
1412 warning_text
+= code
1413 return (True, warning_text
)
1415 def get_hypervisor_information(virsh_conn
, hypervisor
):
1416 type_
= virsh_conn
.getType().rstrip('\n')
1417 version
= virsh_conn
.getVersion()
1418 lib_version
= virsh_conn
.getLibVersion()
1421 tree
=ElementTree
.fromstring(virsh_conn
.getCapabilities())
1422 for target
in tree
.findall("guest"):
1423 os_type
= target
.find("os_type").text
1424 #We only allow full virtualization
1425 if os_type
!= 'hvm':
1427 wordsize
= int(target
.find('arch/wordsize').text
)
1429 for domain
in target
.findall("arch/domain"):
1430 domains
.append(domain
.get("type"))
1432 (return_status
, code
) = hypervisor
.set(type_
, version
, lib_version
, domains
)
1433 if not return_status
:
1434 return (return_status
, code
)
1437 class RADavailableResourcesClass(RADclass
):
1438 def __init__(self
, resources
):
1439 """Copy resources from the RADclass (server resources not taking into account resources used by VMs"""
1441 self
.reserved
= dict() #Dictionary of reserved resources for a server. Key are VNFC names and values RADreservedResources
1442 self
.cores_consumption
= None #Dictionary of cpu consumption. Key is the cpu and the value is
1444 self
.machine
= resources
.machine
1445 self
.user
= resources
.user
1446 self
.password
= resources
.password
1447 self
.name
= resources
.name
1448 self
.nr_processors
= resources
.nr_processors
1449 self
.processor_family
= resources
.processor_family
1450 self
.processor_manufacturer
= resources
.processor_manufacturer
1451 self
.processor_version
= resources
.processor_version
1452 self
.processor_features
= resources
.processor_features
1453 self
.memory_type
= resources
.memory_type
1454 self
.memory_freq
= resources
.memory_freq
1455 self
.memory_nr_channels
= resources
.memory_nr_channels
1456 self
.memory_size
= resources
.memory_size
1457 self
.memory_hugepage_sz
= resources
.memory_hugepage_sz
1458 self
.hypervisor
= Hypervisor()
1459 self
.hypervisor
.assign(resources
.hypervisor
)
1461 self
.os
.assign(resources
.os
)
1463 for node_k
, node_v
in resources
.nodes
.iteritems():
1464 self
.nodes
[node_k
] = Node()
1465 self
.nodes
[node_k
].assign(node_v
)
1468 def _get_cores_consumption_warnings(self
):
1469 """Returns list of warning strings in case warnings are generated.
1470 In case no warnings are generated the return value will be an empty list"""
1472 #Get the cores consumption
1473 (return_status
, code
) = get_ssh_connection(self
.machine
, self
.user
, self
.password
)
1474 if not return_status
:
1475 return (return_status
, code
)
1477 command
= 'mpstat -P ALL 1 1 | grep Average | egrep -v CPU\|all'
1478 (_
, stdout
, stderr
) = ssh_conn
.exec_command(command
)
1479 error
= stderr
.read()
1481 return (False, error
)
1483 self
.cores_consumption
= dict()
1484 for line
in stdout
.readlines():
1485 cpu_usage_split
= re
.split('\t| *', line
.rstrip('\n'))
1486 usage
= 100 *(1 - float(cpu_usage_split
[10]))
1488 self
.cores_consumption
[int(cpu_usage_split
[1])] = usage
1490 #Check if any core marked as available in the nodes has cpu_usage > 0
1491 for _
, node_v
in self
.nodes
.iteritems():
1492 cores
= node_v
.processor
.eligible_cores
1496 if core
in self
.cores_consumption
:
1497 warnings
.append('Warning: Core '+str(core
)+' is supposed to be idle but it is consuming '+str(self
.cores_consumption
[core
])+'%')
1499 if cpu
in self
.cores_consumption
:
1500 warnings
.append('Warning: Core '+str(core
)+' is supposed to be idle but it is consuming '+str(self
.cores_consumption
[cpu
])+'%')
1504 def reserved_to_text(self
):
1506 for VNFC_name
, VNFC_reserved
in self
.reserved
.iteritems():
1507 text
+= ' VNFC: '+str(VNFC_name
)+'\n'
1508 text
+= VNFC_reserved
.to_text()
1512 def obtain_usage(self
):
1514 #Iterate through nodes to get cores, eligible cores, memory and physical ports (save ports usage for next section)
1516 ports_usage
= dict()
1517 hugepage_size
= dict()
1518 for node_k
, node_v
in self
.nodes
.iteritems():
1520 ports_usage
[node_k
] = dict()
1521 eligible_cores
= list()
1522 for pair
in node_v
.processor
.eligible_cores
:
1523 if isinstance(pair
, list):
1524 for element
in pair
:
1525 eligible_cores
.append(element
)
1527 eligible_cores
.append(pair
)
1528 node
['cpus'] = {'cores':node_v
.processor
.cores
,'eligible_cores':eligible_cores
}
1529 node
['memory'] = {'size':str(node_v
.memory
.node_size
/(1024*1024*1024))+'GB','eligible':str(node_v
.memory
.eligible_memory
/(1024*1024*1024))+'GB'}
1530 hugepage_size
[node_k
] = node_v
.memory
.hugepage_sz
1533 for nic
in node_v
.nic_list
:
1534 for port
in nic
.ports
.itervalues():
1535 if port
.enabled
and not port
.virtual
:
1536 ports
[port
.name
] = {'speed':str(port
.speed
/1000000000)+'G'}
1537 # print '*************** ',port.name,'speed',port.speed
1538 ports_usage
[node_k
][port
.name
] = 100 - int(100*float(port
.available_bw
)/float(port
.speed
))
1539 node
['ports'] = ports
1540 nodes
[node_k
] = node
1543 #Iterate through reserved section to get used cores, used memory and port usage
1546 #reserved_cores = list
1547 for node_k
in self
.nodes
.iterkeys():
1548 if not node_k
in cores
:
1549 cores
[node_k
] = list()
1551 for _
, reserved
in self
.reserved
.iteritems():
1552 if node_k
in reserved
.node_reserved_resources
:
1553 node_v
= reserved
.node_reserved_resources
[node_k
]
1554 cores
[node_k
].extend(node_v
.reserved_cores
)
1555 memory
[node_k
] += node_v
.reserved_hugepage_nr
* hugepage_size
[node_k
]
1558 for node_k
in self
.nodes
.iterkeys():
1560 for name
, usage
in ports_usage
[node_k
].iteritems():
1561 ports
[name
] = {'occupied':str(usage
)+'%'}
1562 # print '****************cores',cores
1563 # print '****************memory',memory
1564 occupation
[node_k
] = {'cores':cores
[node_k
],'memory':str(memory
[node_k
]/(1024*1024*1024))+'GB','ports':ports
}
1565 resp
['occupation'] = occupation
1569 class RADreservedResources():
1571 self
.node_reserved_resources
= dict() #dict. keys are the RAD nodes id, values are NodeReservedResources
1572 self
.mgmt_interface_pci
= None #pci in the VNF for the management interface
1573 self
.image
= None #Path in remote machine of the VNFC image
1575 def update(self
,reserved
):
1576 self
.image
= reserved
.image
1577 self
.mgmt_interface_pci
= reserved
.mgmt_interface_pci
1578 for k
,v
in reserved
.node_reserved_resources
.iteritems():
1579 if k
in self
.node_reserved_resources
.keys():
1580 return (False, 'Duplicated node entry '+str(k
)+' in reserved resources')
1581 self
.node_reserved_resources
[k
]=v
1586 text
= ' image: '+str(self
.image
)+'\n'
1587 for node_id
, node_reserved
in self
.node_reserved_resources
.iteritems():
1588 text
+= ' Node ID: '+str(node_id
)+'\n'
1589 text
+= node_reserved
.to_text()
1592 class NodeReservedResources():
1594 # reserved_shared_cores = None #list. List of all cores that the VNFC needs in shared mode #TODO Not used
1595 # reserved_memory = None #Integer. Amount of KiB needed by the VNFC #TODO. Not used since hugepages are used
1596 self
.reserved_cores
= list() #list. List of all cores that the VNFC uses
1597 self
.reserved_hugepage_nr
= 0 #Integer. Number of hugepages needed by the VNFC
1598 self
.reserved_ports
= dict() #dict. The key is the physical port pci and the value the VNFC port description
1599 self
.vlan_tags
= dict()
1600 self
.cpu_pinning
= None
1603 text
= ' cores: '+str(self
.reserved_cores
)+'\n'
1604 text
+= ' cpu_pinning: '+str(self
.cpu_pinning
)+'\n'
1605 text
+= ' hugepages_nr: '+str(self
.reserved_hugepage_nr
)+'\n'
1606 for port_pci
, port_description
in self
.reserved_ports
.iteritems():
1607 text
+= ' port: '+str(port_pci
)+'\n'
1608 text
+= port_description
.to_text()
1611 # def update(self,reserved):
1612 # self.reserved_cores = list(reserved.reserved_cores)
1613 # self.reserved_hugepage_nr = reserved.reserved_hugepage_nr
1614 # self.reserved_ports = dict(reserved.reserved_ports)
1615 # self.cpu_pinning = list(reserved.cpu_pinning)