RADclass.py

   1 # -*- coding: utf-8 -*-
   2 import code
   3
   4 ##
   5 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
   6 # This file is part of openmano
   7 # All Rights Reserved.
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License"); you may
  10 # not use this file except in compliance with the License. You may obtain
  11 # a copy of the License at
  12 #
  13 #         http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  17 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  18 # License for the specific language governing permissions and limitations
  19 # under the License.
  20 #
  21 # For those usages not covered by the Apache License, Version 2.0 please
  22 # contact with: nfvlabs@tid.es
  23 ##
  24
  25 '''
  26 Implement the logic for obtaining compute nodes information
  27 Resource Availability Descriptor
  28 '''
  29 __author__="Pablo Montes"
  30
  31 #TODO: remove warnings, remove unused things
  32
  33 from definitionsClass import definitionsClass
  34 from auxiliary_functions import get_ssh_connection
  35 import libvirt
  36 from xml.etree import ElementTree
  37 import paramiko
  38 import re
  39 import yaml
  40
  41
  42 def getCredentials(creds, data):
  43     """Used as a backup for libvirt.openAuth in order to provide password that came with data,
  44     not used by the moment
  45     """
  46     print "RADclass:getCredentials", creds, data
  47     for cred in creds:
  48         print cred[1] + ": ",
  49         if cred[0] == libvirt.VIR_CRED_AUTHNAME:
  50             cred[4] = data
  51         elif cred[0] == libvirt.VIR_CRED_PASSPHRASE:
  52             cred[4] = data
  53         else:
  54             return -1
  55     return 0
  56
  57 class RADclass():
  58     def __init__(self):
  59         self.name = None
  60         self.machine = None
  61         self.user = None
  62         self.password = None
  63         self.nodes = dict()                 #Dictionary of nodes. Keys are the node id, values are Node() elements
  64         self.nr_processors = None           #Integer. Number of processors in the system
  65         self.processor_family = None        #If all nodes have the same value equal them, otherwise keep as None
  66         self.processor_manufacturer = None  #If all nodes have the same value equal them, otherwise keep as None
  67         self.processor_version = None       #If all nodes have the same value equal them, otherwise keep as None
  68         self.processor_features = None      #If all nodes have the same value equal them, otherwise keep as None
  69         self.memory_type = None             #If all nodes have the same value equal them, otherwise keep as None
  70         self.memory_freq = None             #If all nodes have the same value equal them, otherwise keep as None
  71         self.memory_nr_channels = None      #If all nodes have the same value equal them, otherwise keep as None
  72         self.memory_size = None             #Integer. Sum of the memory in all nodes
  73         self.memory_hugepage_sz = None
  74         self.hypervisor = Hypervisor()      #Hypervisor information
  75         self.os = OpSys()                   #Operating system information
  76         self.ports_list = list()            #List containing all network ports in the node. This is used to avoid having defined multiple times the same port in the system
  77
  78
  79     def obtain_RAD(self, user, password, machine):
  80         """This function obtains the RAD information from the remote server.
  81         It uses both a ssh and a libvirt connection.
  82         It is desirable in future versions get rid of the ssh connection, but currently
  83         libvirt does not provide all the needed information.
  84         Returns (True, Warning) in case of success and (False, <error>) in case of error"""
  85         warning_text=""
  86         try:
  87             #Get virsh and ssh connection
  88             (return_status, code) = get_ssh_connection(machine, user, password)
  89             if not return_status:
  90                 print 'RADclass.obtain_RAD() error:', code
  91                 return (return_status, code)
  92             ssh_conn = code
  93
  94             self.connection_IP = machine
  95             #print "libvirt open pre"
  96             virsh_conn=libvirt.open("qemu+ssh://"+user+'@'+machine+"/system")
  97             #virsh_conn=libvirt.openAuth("qemu+ssh://"+user+'@'+machine+"/system",
  98             #        [[libvirt.VIR_CRED_AUTHNAME, libvirt.VIR_CRED_PASSPHRASE, libvirt.VIR_CRED_USERNAME], getCredentials, password],
  99             #        0)
 100             #print "libvirt open after"
 101
 102     #         #Set connection infomation
 103     #         (return_status, code) = self.set_connection_info(machine, user, password)
 104     #         if not return_status:
 105     #             return (return_status, 'Error in '+machine+': '+code)
 106
 107             #Set server name
 108             machine_name = get_hostname(virsh_conn)
 109             (return_status, code) = self.set_name(machine_name)
 110             if not return_status:
 111                 return (return_status, 'Error at self.set_name in '+machine+': '+code)
 112             warning_text += code
 113
 114             #Get the server processors information
 115             processors = dict()
 116             (return_status, code) = get_processor_information(ssh_conn, virsh_conn, processors)
 117             if not return_status:
 118                 return (return_status, 'Error at get_processor_information in '+machine+': '+code)
 119             warning_text += code
 120
 121             #Get the server memory information
 122             memory_nodes = dict()
 123             (return_status, code) = get_memory_information(ssh_conn, virsh_conn, memory_nodes)
 124             if not return_status:
 125                 return (return_status, 'Error at get_memory_information in '+machine+': '+code)
 126             warning_text += code
 127
 128             #Get nics information
 129             nic_topology = dict()
 130     #         (return_status, code) = get_nic_information_old(ssh_conn, nic_topology)
 131             (return_status, code) = get_nic_information(ssh_conn, virsh_conn, nic_topology)
 132             if not return_status:
 133                 return (return_status, 'Error at get_nic_information in '+machine+': '+code)
 134             warning_text += code
 135
 136             #Pack each processor, memory node  and nics in a node element
 137             #and add the node to the RAD element
 138             for socket_id, processor in processors.iteritems():
 139                 node = Node()
 140                 if not socket_id in nic_topology:
 141                     nic_topology[socket_id] = list()
 142
 143                 (return_status, code) = node.set(processor, memory_nodes[socket_id], nic_topology[socket_id])
 144     #             else:
 145     #                 (return_status, code) = node.set(processor, memory_nodes[socket_id])
 146                 if not return_status:
 147                     return (return_status, 'Error at node.set in '+machine+': '+code)
 148                 warning_text += code
 149                 (return_status, code) = self.insert_node(node)
 150                 if not return_status:
 151                     return (return_status, 'Error at self.insert_node in '+machine+': '+code)
 152                 if code not in warning_text:
 153                     warning_text += code
 154
 155             #Fill os data
 156             os = OpSys()
 157             (return_status, code) = get_os_information(ssh_conn, os)
 158             if not return_status:
 159                 return (return_status, 'Error at get_os_information in '+machine+': '+code)
 160             warning_text += code
 161             (return_status, code) = self.set_os(os)
 162             if not return_status:
 163                 return (return_status, 'Error at self.set_os in '+machine+': '+code)
 164             warning_text += code
 165
 166             #Fill hypervisor data
 167             hypervisor = Hypervisor()
 168             (return_status, code) = get_hypervisor_information(virsh_conn, hypervisor)
 169             if not return_status:
 170                 return (return_status, 'Error at get_hypervisor_information in '+machine+': '+code)
 171             warning_text += code
 172             (return_status, code) = self.set_hypervisor(hypervisor)
 173             if not return_status:
 174                 return (return_status, 'Error at self.set_hypervisor in '+machine+': '+code)
 175             warning_text += code
 176             ssh_conn.close()
 177
 178             return (True, warning_text)
 179         except libvirt.libvirtError, e:
 180             text = e.get_error_message()
 181             print 'RADclass.obtain_RAD() exception:', text
 182             return (False, text)
 183         except paramiko.ssh_exception.SSHException, e:
 184             text = e.args[0]
 185             print  "obtain_RAD ssh Exception:", text
 186             return False, text
 187
 188     def set_name(self,name):
 189         """Sets the machine name.
 190         Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
 191         if not isinstance(name,str):
 192             return (False, 'The variable \'name\' must be text')
 193         self.name = name
 194         return (True, "")
 195
 196     def set_connection_info(self, machine, user, password):
 197         """Sets the connection information.
 198         Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
 199         if not isinstance(machine,str):
 200             return (False, 'The variable \'machine\' must be text')
 201         if not isinstance(user,str):
 202             return (False, 'The variable \'user\' must be text')
 203 #         if not isinstance(password,str):
 204 #             return (False, 'The variable \'password\' must be text')
 205         (self.machine, self.user, self.password) = (machine, user, password)
 206         return (True, "")
 207
 208     def insert_node(self,node):
 209         """Inserts a new node and updates class variables.
 210         Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
 211         if not isinstance(node,Node):
 212             return (False, 'The variable \'node\' must be a Node element')
 213
 214         if node.id_ in self.nodes:
 215             return (False, 'The node is already present in the nodes list.')
 216
 217         #Check if network ports have not been inserted previously as part of another node
 218         for port_key in node.ports_list:
 219             if port_key in self.ports_list:
 220                 return (False, 'Network port '+port_key+' defined multiple times in the system')
 221             self.ports_list.append(port_key)
 222
 223         #Insert the new node
 224         self.nodes[node.id_] = node
 225
 226         #update variables
 227         self.update_variables()
 228
 229         return (True, "")
 230
 231     def update_variables(self):
 232         """Updates class variables.
 233         Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
 234         warning_text=""
 235         #The number of processors and nodes is the same
 236         self.nr_processors = len(self.nodes)
 237
 238         #If all processors are the same get the values. Otherwise keep them as none
 239         prev_processor_family = prev_processor_manufacturer = prev_processor_version = prev_processor_features = None
 240         different_processor_family = different_processor_manufacturer = different_processor_version = different_processor_features = False
 241         for node in self.nodes.itervalues():
 242             (self.processor_family, self.processor_manufacturer, self.processor_version, self.processor_features) = node.get_processor_info()
 243             if prev_processor_family != None and self.processor_family != prev_processor_family:
 244                 different_processor_family = True
 245             if prev_processor_manufacturer != None and self.processor_manufacturer != prev_processor_manufacturer:
 246                 different_processor_manufacturer = True
 247             if prev_processor_version != None and self.processor_version != prev_processor_version:
 248                 different_processor_version = True
 249             if prev_processor_features != None and self.processor_features != prev_processor_features:
 250                 different_processor_features = True
 251             (prev_processor_family, prev_processor_manufacturer, prev_processor_version, prev_processor_features) = (self.processor_family, self.processor_manufacturer, self.processor_version, self.processor_features)
 252
 253         if different_processor_family:
 254             self.processor_family = None
 255         if different_processor_features:
 256             self.processor_features = None
 257         if different_processor_manufacturer:
 258             self.processor_manufacturer = None
 259         if different_processor_version:
 260             self.processor_version = None
 261
 262         #If all memory nodes are the same get the values. Otherwise keep them as none
 263         #Sum the total memory
 264         self.memory_size = 0
 265         different_memory_freq = different_memory_nr_channels = different_memory_type = different_memory_hugepage_sz = False
 266         prev_memory_freq = prev_memory_nr_channels = prev_memory_type = prev_memory_hugepage_sz = None
 267         for node in self.nodes.itervalues():
 268             (self.memory_freq, self.memory_nr_channels, self.memory_type, memory_size, self.memory_hugepage_sz) = node.get_memory_info()
 269             self.memory_size += memory_size
 270             if prev_memory_freq != None and self.memory_freq != prev_memory_freq:
 271                 different_memory_freq = True
 272             if prev_memory_nr_channels != None and self.memory_nr_channels != prev_memory_nr_channels:
 273                 different_memory_nr_channels = True
 274             if prev_memory_type != None and self.memory_type != prev_memory_type:
 275                 different_memory_type = True
 276             if prev_memory_hugepage_sz != None and self.memory_hugepage_sz != prev_memory_hugepage_sz:
 277                 different_memory_hugepage_sz = True
 278             (prev_memory_freq, prev_memory_nr_channels, prev_memory_type, prev_memory_hugepage_sz) = (self.memory_freq, self.memory_nr_channels, self.memory_type, self.memory_hugepage_sz)
 279
 280         if different_memory_freq:
 281             self.memory_freq = None
 282         if different_memory_nr_channels:
 283             self.memory_nr_channels = None
 284         if different_memory_type:
 285             self.memory_type = None
 286         if different_memory_hugepage_sz:
 287             warning_text += 'Detected different hugepages size in different sockets\n'
 288
 289         return (True, warning_text)
 290
 291     def set_hypervisor(self,hypervisor):
 292         """Sets the hypervisor.
 293         Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
 294         if not isinstance(hypervisor,Hypervisor):
 295             return (False, 'The variable \'hypervisor\' must be of class Hypervisor')
 296
 297         self.hypervisor.assign(hypervisor)
 298         return (True, "")
 299
 300     def set_os(self,os):
 301         """Sets the operating system.
 302         Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
 303         if not isinstance(os,OpSys):
 304             return (False, 'The variable \'os\' must be of class OpSys')
 305
 306         self.os.assign(os)
 307         return (True, "")
 308
 309     def to_text(self):
 310         text= 'name: '+str(self.name)+'\n'
 311         text+= 'processor:\n'
 312         text+= '    nr_processors: '+str(self.nr_processors)+'\n'
 313         text+= '    family: '+str(self.processor_family)+'\n'
 314         text+= '    manufacturer: '+str(self.processor_manufacturer)+'\n'
 315         text+= '    version: '+str(self.processor_version)+'\n'
 316         text+= '    features: '+str(self.processor_features)+'\n'
 317         text+= 'memory:\n'
 318         text+= '    type: '+str(self.memory_type)+'\n'
 319         text+= '    freq: '+str(self.memory_freq)+'\n'
 320         text+= '    nr_channels: '+str(self.memory_nr_channels)+'\n'
 321         text+= '    size: '+str(self.memory_size)+'\n'
 322         text+= 'hypervisor:\n'
 323         text+= self.hypervisor.to_text()
 324         text+= 'os:\n'
 325         text+= self.os.to_text()
 326         text+= 'resource topology:\n'
 327         text+= '    nr_nodes: '+ str(len(self.nodes))+'\n'
 328         text+= '    nodes:\n'
 329         for node_k, node_v in self.nodes.iteritems():
 330             text+= '        node'+str(node_k)+':\n'
 331             text+= node_v.to_text()
 332         return text
 333
 334     def to_yaml(self):
 335         return yaml.load(self.to_text())
 336
 337 class Node():
 338     def __init__(self):
 339         self.id_ = None                      #Integer. Node id. Unique in the system
 340         self.processor = ProcessorNode()    #Information about the processor in the node
 341         self.memory = MemoryNode()          #Information about the memory in the node
 342         self.nic_list = list()              #List of Nic() containing information about the nics associated to the node
 343         self.ports_list = list()            #List containing all network ports in the node. This is used to avoid having defined multiple times the same port in the system
 344
 345     def get_processor_info(self):
 346         """Gets the processor information. Returns (processor_family, processor_manufacturer, processor_version, processor_features)"""
 347         return self.processor.get_info()
 348
 349     def get_memory_info(self):
 350         """Gets the memory information. Returns (memory_freq, memory_nr_channels, memory_type, memory_size)"""
 351         return self.memory.get_info()
 352
 353 #     def set(self, *args):
 354 #         """Sets the node information. Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
 355 #         if len(args)==2:
 356 #             processor = args[0]
 357 #             memory = args[1]
 358 #             nics = False
 359 #         elif len(args)==3:
 360 #             processor = args[0]
 361 #             memory = args[1]
 362 #             nic_list = args[2]
 363 #             nics = True
 364 #         else:
 365 #             return (False, 'Wrong number of elements calling Node().set()')
 366
 367     def set(self, processor, memory, nic_list):
 368         (status, return_code) = self.processor.assign(processor)
 369         if not status:
 370             return (status, return_code)
 371
 372         self.id_ = processor.id_
 373
 374         (status, return_code) = self.memory.assign(memory)
 375         if not status:
 376             return (status, return_code)
 377
 378 #         if nics:
 379         for nic in nic_list:
 380             if not isinstance(nic,Nic):
 381                 return (False, 'The nics must be of type Nic')
 382             self.nic_list.append(nic)
 383             for port_key in nic.ports.iterkeys():
 384                 if port_key in self.ports_list:
 385                     return (False, 'Network port '+port_key+'defined multiple times in the same node')
 386                 self.ports_list.append(port_key)
 387
 388         return (True,"")
 389
 390     def assign(self, node):
 391         """Sets the node information.
 392         Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
 393         warning_text=""
 394         processor = node.processor
 395         memory = node.memory
 396         nic_list = node.nic_list
 397         (status, return_code) = self.processor.assign(processor)
 398         if not status:
 399             return (status, return_code)
 400
 401         self.id_ = processor.id_
 402
 403         (status, return_code) = self.memory.assign(memory)
 404         if not status:
 405             return (status, return_code)
 406         warning_text += code
 407
 408         for nic in nic_list:
 409             if not isinstance(nic,Nic):
 410                 return (False, 'The nics must be of type Nic')
 411             self.nic_list.append(nic)
 412             for port_key in nic.ports.iterkeys():
 413                 if port_key in self.ports_list:
 414                     return (False, 'Network port '+port_key+'defined multiple times in the same node')
 415                 self.ports_list.append(port_key)
 416
 417         return (True,warning_text)
 418
 419     def to_text(self):
 420         text= '            id: '+str(self.id_)+'\n'
 421         text+= '            cpu:\n'
 422         text += self.processor.to_text()
 423         text+= '            memory:\n'
 424         text += self.memory.to_text()
 425         if len(self.nic_list) > 0:
 426             text+= '            nics:\n'
 427             nic_index = 0
 428             for nic in self.nic_list:
 429                 text+= '                nic '+str(nic_index)+':\n'
 430                 text += nic.to_text()
 431                 nic_index += 1
 432         return text
 433
 434 class ProcessorNode():
 435     #Definition of the possible values of processor variables
 436     possible_features = definitionsClass.processor_possible_features
 437     possible_manufacturers = definitionsClass.processor_possible_manufacturers
 438     possible_families = definitionsClass.processor_possible_families
 439     possible_versions = definitionsClass.processor_possible_versions
 440
 441     def __init__(self):
 442         self.id_ = None              #Integer. Numeric identifier of the socket
 443         self.family = None          #Text. Family name of the processor
 444         self.manufacturer = None    #Text. Manufacturer of the processor
 445         self.version = None         #Text. Model version of the processor
 446         self.features = list()      #list. List of features offered by the processor
 447         self.cores = list()         #list. List of cores in the processor. In case of hyperthreading the coupled cores are expressed as [a,b]
 448         self.eligible_cores = list()#list. List of cores that can be used
 449         #self.decicated_cores
 450         #self.shared_cores -> this should also contain information to know if cores are being used
 451
 452     def assign(self, processor):
 453         """Sets the processor information.
 454         Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
 455         if not isinstance(processor,ProcessorNode):
 456             return (False, 'The variable \'processor\' must be of class ProcessorNode')
 457
 458         self.id_ = processor.id_
 459         self.family = processor.family
 460         self.manufacturer = processor.manufacturer
 461         self.version = processor.version
 462         self.features = processor.features
 463         self.cores = processor.cores
 464         self.eligible_cores = processor.eligible_cores
 465
 466         return (True, "")
 467
 468     def set(self, id_, family, manufacturer, version, features, cores):
 469         """Sets the processor information.
 470         Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
 471         warning_text = ""
 472
 473         if not isinstance(id_,int):
 474             return (False, 'The processor id_ must be of type int')
 475         if not isinstance(family,str):
 476             return (False, 'The processor family must be of type str')
 477         if not isinstance(manufacturer,str):
 478             return (False, 'The processor manufacturer must be of type str')
 479         if not isinstance(version,str):
 480             return (False, 'The processor version must be of type str')
 481         if not isinstance(features,list):
 482             return (False, 'The processor features must be of type list')
 483         if not isinstance(cores,list):
 484             return (False, 'The processor cores must be of type list')
 485
 486         (self.id_, self.family, self.manufacturer, self.version) = (id_, family, manufacturer, version)
 487
 488         if not manufacturer in self.possible_manufacturers:
 489             warning_text += "processor manufacturer '%s' not among: %s\n" %(manufacturer, str(self.possible_manufacturers))
 490         if not family in self.possible_families:
 491             warning_text += "family '%s' not among: %s\n" % (family, str(self.possible_families))
 492 #        if not version in self.possible_versions:
 493 #            warning_text += 'The version %s is not one of these: %s\n' % (version, str(self.possible_versions))
 494
 495         for feature in features:
 496             if not feature in self.possible_features:
 497                 warning_text += "processor feature '%s' not among: %s\n" % (feature, str(self.possible_versions))
 498             self.features.append(feature)
 499
 500         for iterator in sorted(cores):
 501             if not isinstance(iterator,list) or not all(isinstance(x, int) for x in iterator):
 502                 return (False, 'The cores list must be in the form of [[a,b],[c,d],...] where a,b,c,d are of type int')
 503             self.cores.append(iterator)
 504
 505         self.set_eligible_cores()
 506
 507         return (True,warning_text)
 508
 509     def set_eligible_cores(self):
 510         """Set the default eligible cores, this is all cores non used by the host operating system"""
 511         not_first = False
 512         for iterator in self.cores:
 513             if not_first:
 514                 self.eligible_cores.append(iterator)
 515             else:
 516                 not_first = True
 517         return
 518
 519     def get_info(self):
 520         """Returns processor parameters (self.family, self.manufacturer, self.version, self.features)"""
 521         return (self.family, self.manufacturer, self.version, self.features)
 522
 523     def to_text(self):
 524         text= '                id: '+str(self.id_)+'\n'
 525         text+= '                family: '+self.family+'\n'
 526         text+= '                manufacturer: '+self.manufacturer+'\n'
 527         text+= '                version: '+self.version+'\n'
 528         text+= '                features: '+str(self.features)+'\n'
 529         text+= '                cores: '+str(self.cores)+'\n'
 530         text+= '                eligible_cores: '+str(self.eligible_cores)+'\n'
 531         return text
 532
 533 class MemoryNode():
 534     def __init__(self):
 535         self.modules = list()               #List of MemoryModule(). List of all modules installed in the node
 536         self.nr_channels = None             #Integer. Number of modules installed in the node
 537         self.node_size = None               #Integer. Total size in KiB of memory installed in the node
 538         self.eligible_memory = None         #Integer. Size in KiB of eligible memory in the node
 539         self.hugepage_sz = None             #Integer. Size in KiB of hugepages
 540         self.hugepage_nr = None             #Integer. Number of hugepages allocated in the module
 541         self.eligible_hugepage_nr = None    #Integer. Number of eligible hugepages in the node
 542         self.type_ = None                    #Text. Type of memory modules. If modules have a different value keep it as None
 543         self.freq = None                    #Integer. Frequency of the modules in MHz. If modules have a different value keep it as None
 544         self.module_size = None             #Integer. Size of the modules in KiB. If modules have a different value keep it as None
 545         self.form_factor = None             #Text. Form factor of the modules. If modules have a different value keep it as None
 546
 547     def assign(self, memory_node):
 548         return self.set(memory_node.modules, memory_node.hugepage_sz, memory_node.hugepage_nr)
 549
 550     def set(self, modules, hugepage_sz, hugepage_nr):
 551         """Set the memory node information. hugepage_sz must be expressed in KiB.
 552         Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
 553         if not isinstance(modules, list):
 554             return (False, 'The modules must be a list of elements of class MemoryModule')
 555         if not isinstance(hugepage_sz,int):
 556             return (False, 'The hugepage_sz variable must be an int expressing the size in KiB')
 557         if not isinstance(hugepage_nr,int):
 558             return (False, 'The hugepage_nr variable must be of type int')
 559
 560         (self.hugepage_sz, self.hugepage_nr) = (hugepage_sz, hugepage_nr)
 561         self.node_size = self.nr_channels = 0
 562
 563         different_type = different_freq = different_module_size = different_form_factor = False
 564         prev_type = prev_freq = prev_module_size = prev_form_factor = None
 565         for iterator in modules:
 566             if not isinstance(iterator,MemoryModule):
 567                 return (False, 'The modules must be a list of elements of class MemoryModule')
 568             self.modules.append(iterator)
 569             (self.type_, self.freq, self.module_size, self.form_factor) = (iterator.type_, iterator.freq, iterator.size, iterator.form_factor)
 570             self.node_size += self.module_size
 571             self.nr_channels += 1
 572             if prev_type != None and prev_type != self.type_:
 573                 different_type = True
 574             if prev_freq != None and prev_freq != self.freq:
 575                 different_freq = True
 576             if prev_module_size != None and prev_module_size != self.module_size:
 577                 different_module_size = True
 578             if prev_form_factor != None and prev_form_factor != self.form_factor:
 579                 different_form_factor = True
 580             (prev_type, prev_freq, prev_module_size, prev_form_factor) = (self.type_, self.freq, self.module_size, self.form_factor)
 581
 582         if different_type:
 583             self.type_ = None
 584         if different_freq:
 585             self.freq = None
 586         if different_module_size:
 587             self.module_size = None
 588         if different_form_factor:
 589             self.form_factor = None
 590
 591         (return_value, error_code) = self.set_eligible_memory()
 592         if not return_value:
 593             return (return_value, error_code)
 594
 595         return (True, "")
 596
 597     def set_eligible_memory(self):
 598         """Sets the default eligible_memory and eligible_hugepage_nr. This is all memory but 2GiB and all hugepages"""
 599         self.eligible_memory = self.node_size - 2*1024*1024
 600         if self.eligible_memory < 0:
 601             return (False, "There is less than 2GiB of memory in the module")
 602
 603         self.eligible_hugepage_nr = self.hugepage_nr
 604         return (True,"")
 605
 606     def get_info(self):
 607         """Return memory information (self.freq, self.nr_channels, self.type_, self.node_size)"""
 608         return (self.freq, self.nr_channels, self.type_, self.node_size, self.hugepage_sz)
 609
 610     def to_text(self):
 611         text= '                node_size: '+str(self.node_size)+'\n'
 612         text+= '                nr_channels: '+str(self.nr_channels)+'\n'
 613         text+= '                eligible_memory: '+str(self.eligible_memory)+'\n'
 614         text+= '                hugepage_sz: '+str(self.hugepage_sz)+'\n'
 615         text+= '                hugepage_nr: '+str(self.hugepage_nr)+'\n'
 616         text+= '                eligible_hugepage_nr: '+str(self.eligible_hugepage_nr)+'\n'
 617         text+= '                type: '+self.type_+'\n'
 618         text+= '                freq: '+str(self.freq)+'\n'
 619         text+= '                module_size: '+str(self.module_size)+'\n'
 620         text+= '                form_factor: '+self.form_factor+'\n'
 621         text+= '                modules details:\n'
 622         for module in self.modules:
 623             text += module.to_text()
 624         return text
 625
 626 class MemoryModule():
 627     #Definition of the possible values of module variables
 628     possible_types = definitionsClass.memory_possible_types
 629     possible_form_factors = definitionsClass.memory_possible_form_factors
 630
 631     def __init__(self):
 632         self.locator = None     #Text. Name of the memory module
 633         self.type_ = None        #Text. Type of memory module
 634         self.freq = None        #Integer. Frequency of the module in MHz
 635         self.size = None        #Integer. Size of the module in KiB
 636         self.form_factor = None #Text. Form factor of the module
 637
 638     def set(self, locator, type_, freq, size, form_factor):
 639         """Sets the memory module information.
 640         Frequency must be expressed in MHz and size in KiB.
 641         Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
 642         warning_text=""
 643         if not isinstance(locator, str):
 644             return (False, "The type of the variable locator must be str")
 645         if not isinstance(type_, str):
 646             return (False, "The type of the variable type_ must be str")
 647         if not isinstance(form_factor, str):
 648             return (False, "The type of the variable form_factor must be str")
 649         if not isinstance(freq, int):
 650             return (False, "The type of the variable freq must be int")
 651         if not isinstance(size, int):
 652             return (False, "The type of the variable size must be int")
 653
 654         if not form_factor in self.possible_form_factors:
 655             warning_text += "memory form_factor '%s' not among: %s\n" %(form_factor, str(self.possible_form_factors))
 656         if not type_ in self.possible_types:
 657             warning_text += "memory type '%s' not among: %s\n" %(type_, str(self.possible_types))
 658
 659         (self.locator, self.type_, self.freq, self.size, self.form_factor) = (locator, type_, freq, size, form_factor)
 660         return (True, warning_text)
 661
 662     def to_text(self):
 663         text= '                    '+self.locator+':\n'
 664         text+= '                        type: '+self.type_+'\n'
 665         text+= '                        freq: '+str(self.freq)+'\n'
 666         text+= '                        size: '+str(self.size)+'\n'
 667         text+= '                        form factor: '+self.form_factor+'\n'
 668         return text
 669
 670 class Nic():
 671     def __init__(self):
 672         self.model = None       #Text. Model of the nic
 673         self.ports = dict()     #Dictionary of ports. Keys are the port name, value are Port() elements
 674
 675     def set_model(self, model):
 676         """Sets the model of the nic. Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
 677         if not isinstance(model,str):
 678             return (False, 'The \'model\' must be of type str')
 679
 680         self.model = model
 681         return (True, "")
 682
 683     def add_port(self, port):
 684         """Adds a port to the nic. Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
 685         if not isinstance(port,Port):
 686             return (False, 'The \'port\' must be of class Port')
 687
 688 #        port_id = str(port.pci_device_id[0])+':'+str(port.pci_device_id[1])+':'+str(port.pci_device_id[2])+'.'+str(port.pci_device_id[3])
 689 #CHANGED
 690 #        port_id = port.name
 691         port_id = port.pci_device_id
 692 #CHANGED END
 693         if port_id in self.ports:
 694             return (False, 'The \'port\' '+port.pci_device_id+' is duplicated in the nic')
 695 #             return (False, 'The \'port\' is duplicated in the nic')
 696
 697         self.ports[port_id] = port
 698         return (True, "")
 699
 700     def to_text(self):
 701         text= '                    model: '+ str(self.model)+'\n'
 702         text+= '                    ports: '+'\n'
 703         for key,port in self.ports.iteritems():
 704             text+= '                        "'+key+'":'+'\n'
 705             text += port.to_text()
 706         return text
 707
 708 class Port():
 709     def __init__(self):
 710         self.name = None            #Text. Port name
 711         self.virtual = None         #Boolean. States if the port is a virtual function
 712         self.enabled = None         #Boolean. States if the port is enabled
 713         self.eligible = None        #Boolean. States if the port is eligible
 714         self.speed = None           #Integer. Indicates the speed in Mbps
 715         self.available_bw = None    #Integer. BW in Mbps that is available.
 716         self.mac = None             #list. Indicates the mac address of the port as a list in format ['XX','XX','XX','XX','XX','XX']
 717         self.pci_device_id_split = None   #list. Indicates the pci address  of the port as a list in format ['XXXX','XX','XX','X']
 718         self.pci_device_id = None
 719         self.PF_pci_device_id = None
 720
 721 #     def set(self, name, virtual, enabled, speed, mac, pci_device_id, pci_device_id_split):
 722 #         """Sets the port information. The variable speed indicates the speed in Mbps. Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
 723 #         if not isinstance(name,str):
 724 #             return (False, 'The variable \'name\' must be of type str')
 725 #         if not isinstance(virtual,bool):
 726 #             return (False, 'The variable \'virtual\' must be of type bool')
 727 #         if not isinstance(enabled,bool):
 728 #             return (False, 'The variable \'enabled\' must be of type bool')
 729 #         if not isinstance(enabled,bool):
 730 #             return (speed, 'The variable \'speed\' must be of type int')
 731 #         if not isinstance(mac, list) and not isinstance(mac,NoneType):
 732 #             return (False, 'The variable \'enabled\' must be of type list indicating the mac address in format [\'XXXX\',\'XX\',\'XX\',\'X\'] or NoneType')
 733 #         if not isinstance(pci_device_id_split, list) or len(pci_device_id_split) != 4:
 734 #             return (False, 'The variable \'pci_device_id_split\' must be of type list, indicating the pci address in format [\'XX\',\'XX\',\'XX\',\'XX\',\'XX\',\'XX\']')
 735 #
 736 #         expected_len = [4,2,2,1]
 737 #         index = 0
 738 #         for iterator in pci_device_id_split:
 739 #             if not isinstance(iterator,str) or not iterator.isdigit() or len(iterator) != expected_len[index]:
 740 #                 return (False, 'The variable \'pci_device_id_split\' must be of type list, indicating the pci address in format [\'XX\',\'XX\',\'XX\',\'XX\',\'XX\',\'XX\']')
 741 #             index += 1
 742 #
 743 #         if not isinstance(mac,NoneType):
 744 #             for iterator in mac:
 745 #                 if not isinstance(iterator,str) or not iterator.isalnum() or len(iterator) != 2:
 746 #                     return (False, 'The variable \'enabled\' must be of type list indicating the mac address in format [\'XXXX\',\'XX\',\'XX\',\'X\'] or NoneType')
 747 #
 748 #         #By default only virtual ports are eligible
 749 # #         (self.name, self.virtual, self.enabled, self.eligible, self.available_bw, self.speed, self.mac, self.pci_device_id, self.pci_device_id_split) = (name, virtual, enabled, virtual, speed, speed, mac, pci_device_id, pci_device_id_split)
 750 #         (self.name, self.virtual, self.enabled, self.eligible, self.available_bw, self.mac, self.pci_device_id, self.pci_device_id_split) = (name, virtual, enabled, virtual, speed, mac, pci_device_id, pci_device_id_split)
 751
 752     def to_text(self):
 753         text= '                            pci: "'+ str(self.pci_device_id)+'"\n'
 754         text+= '                            virtual: '+ str(self.virtual)+'\n'
 755         if self.virtual:
 756             text+= '                            PF_pci_id: "'+self.PF_pci_device_id+'"\n'
 757         text+= '                            eligible: '+ str(self.eligible)+'\n'
 758         text+= '                            enabled: '+str(self.enabled)+'\n'
 759         text+= '                            speed: '+ str(self.speed)+'\n'
 760         text+= '                            available bw: '+ str(self.available_bw)+'\n'
 761         text+= '                            mac: '+ str(self.mac)+'\n'
 762         text+= '                            source_name: '+ str(self.name)+'\n'
 763         return text
 764
 765 class Hypervisor():
 766     #Definition of the possible values of hypervisor variables
 767     possible_types = definitionsClass.hypervisor_possible_types
 768     possible_domain_types = definitionsClass.hypervisor_possible_domain_types
 769
 770     def __init__(self):
 771         self.type_ = None            #Text. Hypervisor type_
 772         self.version = None         #int. Hypervisor version
 773         self.lib_version = None     #int. Libvirt version used to compile hypervisor
 774         self.domains = list()       #list. List of all the available domains
 775
 776     def set(self, hypervisor, version, lib_version, domains):
 777         warning_text=""
 778         if not isinstance(hypervisor,str):
 779             return (False, 'The variable type_ must be of type str')
 780         if not isinstance(version,int):
 781             return (False, 'The variable version must be of type int')
 782         if not isinstance(lib_version,int):
 783             return (False, 'The library version must be of type int')
 784         if not isinstance(domains,list):
 785             return (False, 'Domains must be a list of the possible domains as str')
 786
 787         if not hypervisor in self.possible_types:
 788             warning_text += "Hyperpivor '%s' not among: %s\n" % (hypervisor, str(self.possible_types))
 789
 790         valid_domain_found = False
 791         for domain in domains:
 792             if not isinstance(domain,str):
 793                 return (False, 'Domains must be a list of the possible domains as str')
 794             if domain in self.possible_domain_types:
 795                 valid_domain_found = True
 796                 self.domains.append(domain)
 797
 798         if not valid_domain_found:
 799             warning_text += 'No valid domain found among: %s\n' % str(self.possible_domain_types)
 800
 801
 802         (self.version, self.lib_version, self.type_) = (version, lib_version, hypervisor)
 803         return (True, warning_text)
 804
 805     def assign(self, hypervisor):
 806         (self.version, self.lib_version, self.type_) = (hypervisor.version, hypervisor.lib_version, hypervisor.type_)
 807         for domain in hypervisor.domains:
 808             self.domains.append(domain)
 809         return
 810
 811     def to_text(self):
 812         text= '    type: '+self.type_+'\n'
 813         text+= '    version: '+str(self.version)+'\n'
 814         text+= '    libvirt version: '+ str(self.lib_version)+'\n'
 815         text+= '    domains: '+str(self.domains)+'\n'
 816         return text
 817
 818 class OpSys():
 819     #Definition of the possible values of os variables
 820     possible_id = definitionsClass.os_possible_id
 821     possible_types = definitionsClass.os_possible_types
 822     possible_architectures = definitionsClass.os_possible_architectures
 823
 824     def __init__(self):
 825         self.id_ = None                   #Text. Identifier of the OS. Formed by <Distibutor ID>-<Release>-<Codename>. In linux this can be obtained using lsb_release -a
 826         self.type_ = None                 #Text. Type of operating system
 827         self.bit_architecture = None     #Integer. Architecture
 828
 829     def set(self, id_, type_, bit_architecture):
 830         warning_text=""
 831         if not isinstance(type_,str):
 832             return (False, 'The variable type_ must be of type str')
 833         if not isinstance(id_,str):
 834             return (False, 'The variable id_ must be of type str')
 835         if not isinstance(bit_architecture,str):
 836             return (False, 'The variable bit_architecture must be of type str')
 837
 838         if not type_ in self.possible_types:
 839             warning_text += "os type '%s' not among: %s\n" %(type_, str(self.possible_types))
 840         if not id_ in self.possible_id:
 841             warning_text += "os release '%s' not among: %s\n" %(id_, str(self.possible_id))
 842         if not bit_architecture in self.possible_architectures:
 843             warning_text += "os bit_architecture '%s' not among: %s\n" % (bit_architecture, str(self.possible_architectures))
 844
 845         (self.id_, self.type_, self.bit_architecture) = (id_, type_, bit_architecture)
 846         return (True, warning_text)
 847
 848     def assign(self,os):
 849         (self.id_, self.type_, self.bit_architecture) = (os.id_, os.type_, os.bit_architecture)
 850         return
 851
 852     def to_text(self):
 853         text= '    id: '+self.id_+'\n'
 854         text+= '    type: '+self.type_+'\n'
 855         text+= '    bit_architecture: '+self.bit_architecture+'\n'
 856         return text
 857
 858 def get_hostname(virsh_conn):
 859     return virsh_conn.getHostname().rstrip('\n')
 860
 861 def get_hugepage_size(ssh_conn):
 862     command = 'sudo hugeadm --page-sizes'
 863 #  command = 'hugeadm --page-sizes-all'
 864     (_, stdout, stderr) = ssh_conn.exec_command(command)
 865     error = stderr.read()
 866     if len(error)>0:
 867         raise paramiko.ssh_exception.SSHException(command +' : '+ error)
 868     mem=stdout.read()
 869     if mem=="":
 870         return 0
 871     return int(mem)
 872
 873 def get_hugepage_nr(ssh_conn,hugepage_sz, node_id):
 874     command = 'cat /sys/devices/system/node/node'+str(node_id)+'/hugepages/hugepages-'+str(hugepage_sz/1024)+'kB/nr_hugepages'
 875     (_, stdout, _) = ssh_conn.exec_command(command)
 876     #print command,
 877     #text = stdout.read()
 878     #print "'"+text+"'"
 879     #return int(text)
 880
 881     try:
 882         value=int(stdout.read())
 883     except:
 884         value=0
 885     return value
 886
 887 def get_memory_information(ssh_conn, virsh_conn, memory_nodes):
 888     warning_text=""
 889     tree=ElementTree.fromstring(virsh_conn.getSysinfo(0))
 890     memory_dict = dict()
 891     node_id = 0 #TODO revise. Added for allowing VM as compute hosts
 892     for target in tree.findall("memory_device"):
 893         locator_f = size_f = freq_f = type_f = formfactor_f = False
 894         locator_f = True #TODO revise. Added for allowing VM as compute hosts
 895         module_form_factor = ""
 896         for entry in target.findall("entry"):
 897             if entry.get("name") == 'size':
 898                 size_f = True
 899                 size_split = entry.text.split(' ')
 900                 if size_split[1] == 'MB':
 901                     module_size = int(size_split[0]) * 1024 * 1024
 902                 elif size_split[1] == 'GB':
 903                     module_size = int(size_split[0]) * 1024 * 1024 * 1024
 904                 elif size_split[1] == 'KB':
 905                     module_size = int(size_split[0]) * 1024
 906                 else:
 907                     module_size = int(size_split[0])
 908
 909             elif entry.get("name") == 'speed':
 910                 freq_f = True
 911                 freq_split = entry.text.split(' ')
 912                 if freq_split[1] == 'MHz':
 913                     module_freq = int(freq_split[0]) * 1024 * 1024
 914                 elif freq_split[1] == 'GHz':
 915                     module_freq = int(freq_split[0]) * 1024 * 1024 * 1024
 916                 elif freq_split[1] == 'KHz':
 917                     module_freq = int(freq_split[0]) * 1024
 918
 919             elif entry.get("name") == 'type':
 920                 type_f = True
 921                 module_type = entry.text
 922
 923             elif entry.get("name") == 'form_factor':
 924                 formfactor_f = True
 925                 module_form_factor = entry.text
 926             #TODO revise. Commented for allowing VM as compute hosts
 927             # elif entry.get("name") == 'locator' and not locator_f:
 928             #     # other case, it is obtained by bank_locator that we give priority to
 929             #     locator = entry.text
 930             #     pos = locator.find(module_form_factor)
 931             #     if module_form_factor == locator[0:len(module_form_factor) ]:
 932             #         pos = len(module_form_factor) +1
 933             #     else:
 934             #         pos = 0
 935             #     if locator[pos] in "ABCDEFGH":
 936             #         locator_f = True
 937             #         node_id = ord(locator[pos])-ord('A')
 938             #         #print entry.text, node_id
 939             #
 940             # elif entry.get("name") == 'bank_locator':
 941             #     locator = entry.text
 942             #     pos = locator.find("NODE ")
 943             #     if pos >= 0 and len(locator)>pos+5:
 944             #         if locator[pos+5] in ("01234567"): #len("NODE ") is 5
 945             #             node_id = int(locator[pos+5])
 946             #             locator_f = True
 947             #
 948
 949         #When all module fields have been found add a new module to the list
 950         if locator_f and size_f and freq_f and type_f and formfactor_f:
 951             #If the memory node has not yet been created create it
 952             if node_id not in memory_dict:
 953                 memory_dict[node_id] = []
 954
 955             #Add a new module to the memory node
 956             module = MemoryModule()
 957             #TODO revise. Changed for allowing VM as compute hosts
 958             (return_status, code) = module.set('NODE %d' % node_id, module_type, module_freq, module_size, module_form_factor)
 959             #(return_status, code) = module.set(locator, module_type, module_freq, module_size, module_form_factor)
 960             if not return_status:
 961                 return (return_status, code)
 962             memory_dict[node_id].append(module)
 963             if code not in warning_text:
 964                 warning_text += code
 965             node_id += 1 #TODO revise. Added for allowing VM as compute hosts
 966
 967     #Fill memory nodes
 968     #Hugepage size is constant for all nodes
 969     hugepage_sz = get_hugepage_size(ssh_conn)
 970     for node_id, modules in memory_dict.iteritems():
 971         memory_node = MemoryNode()
 972         memory_node.set(modules, hugepage_sz, get_hugepage_nr(ssh_conn,hugepage_sz, node_id))
 973         memory_nodes[node_id] = memory_node
 974
 975     return (True, warning_text)
 976
 977 def get_cpu_topology_ht(ssh_conn, topology):
 978     command = 'cat /proc/cpuinfo'
 979     (_, stdout, stderr) = ssh_conn.exec_command(command)
 980     error = stderr.read()
 981     if len(error)>0:
 982         raise paramiko.ssh_exception.SSHException(command +' : '+ error)
 983     sockets = []
 984     cores = []
 985     core_map = {}
 986     core_details = []
 987     core_lines = {}
 988     for line in stdout.readlines():
 989         if len(line.strip()) != 0:
 990             name, value = line.split(":", 1)
 991             core_lines[name.strip()] = value.strip()
 992         else:
 993             core_details.append(core_lines)
 994             core_lines = {}
 995
 996     for core in core_details:
 997         for field in ["processor", "core id", "physical id"]:
 998             if field not in core:
 999                 return(False,'Error getting '+field+' value from /proc/cpuinfo')
1000             core[field] = int(core[field])
1001
1002         if core["core id"] not in cores:
1003             cores.append(core["core id"])
1004         if core["physical id"] not in sockets:
1005             sockets.append(core["physical id"])
1006         key = (core["physical id"], core["core id"])
1007         if key not in core_map:
1008             core_map[key] = []
1009         core_map[key].append(core["processor"])
1010
1011     for s in sockets:
1012         hyperthreaded_cores = list()
1013         for c in cores:
1014             hyperthreaded_cores.append(core_map[(s,c)])
1015         topology[s] = hyperthreaded_cores
1016
1017     return (True, "")
1018
1019 def get_processor_information(ssh_conn, vish_conn, processors):
1020     warning_text=""
1021     #Processor features are the same for all processors
1022     #TODO (at least using virsh capabilities)nr_numa_nodes
1023     capabilities = list()
1024     tree=ElementTree.fromstring(vish_conn.getCapabilities())
1025     for target in tree.findall("host/cpu/feature"):
1026         if target.get("name") == 'pdpe1gb':
1027             capabilities.append('lps')
1028         elif target.get("name") == 'dca':
1029             capabilities.append('dioc')
1030         elif target.get("name") == 'vmx' or target.get("name") == 'svm':
1031             capabilities.append('hwsv')
1032         elif target.get("name") == 'ht':
1033             capabilities.append('ht')
1034
1035     target = tree.find("host/cpu/arch")
1036     if target.text == 'x86_64' or target.text == 'amd64':
1037         capabilities.append('64b')
1038
1039     command = 'cat /proc/cpuinfo | grep flags'
1040     (_, stdout, stderr) = ssh_conn.exec_command(command)
1041     error = stderr.read()
1042     if len(error)>0:
1043         raise paramiko.ssh_exception.SSHException(command +' : '+ error)
1044     line = stdout.readline()
1045     if 'ept' in line or 'npt' in line:
1046         capabilities.append('tlbps')
1047
1048     #Find out if IOMMU is enabled
1049     command = 'dmesg |grep -e Intel-IOMMU'
1050     (_, stdout, stderr) = ssh_conn.exec_command(command)
1051     error = stderr.read()
1052     if len(error)>0:
1053         raise paramiko.ssh_exception.SSHException(command +' : '+ error)
1054     if 'enabled' in stdout.read():
1055         capabilities.append('iommu')
1056
1057     #Equivalent for AMD
1058     command = 'dmesg |grep -e AMD-Vi'
1059     (_, stdout, stderr) = ssh_conn.exec_command(command)
1060     error = stderr.read()
1061     if len(error)>0:
1062         raise paramiko.ssh_exception.SSHException(command +' : '+ error)
1063     if len(stdout.read()) > 0:
1064         capabilities.append('iommu')
1065
1066     #-----------------------------------------------------------
1067     topology = dict()
1068     #In case hyperthreading is active it is necessary to determine cpu topology using /proc/cpuinfo
1069     if 'ht' in capabilities:
1070         (return_status, code) = get_cpu_topology_ht(ssh_conn, topology)
1071         if not return_status:
1072             return (return_status, code)
1073         warning_text += code
1074
1075     #Otherwise it is possible to do it using virsh capabilities
1076     else:
1077         for target in tree.findall("host/topology/cells/cell"):
1078             socket_id = int(target.get("id"))
1079             topology[socket_id] = list()
1080             for cpu in target.findall("cpus/cpu"):
1081                 topology[socket_id].append(int(cpu.get("id")))
1082
1083     #-----------------------------------------------------------
1084     #Create a dictionary with the information of all processors
1085     #p_fam = p_man = p_ver = None
1086     tree=ElementTree.fromstring(vish_conn.getSysinfo(0))
1087     #print vish_conn.getSysinfo(0)
1088     #return (False, 'forces error for debuging')
1089     not_populated=False
1090     socket_id = -1     #in case we can not determine the socket_id we assume incremental order, starting by 0
1091     for target in tree.findall("processor"):
1092         count = 0
1093         socket_id += 1
1094         #Get processor id, family, manufacturer and version
1095         for entry in target.findall("entry"):
1096             if entry.get("name") == "status":
1097                 if entry.text[0:11] == "Unpopulated":
1098                     not_populated=True
1099             elif entry.get("name") == 'socket_destination':
1100                 socket_text = entry.text
1101                 if socket_text.startswith('CPU'):
1102                     socket_text = socket_text.strip('CPU')
1103                     socket_text = socket_text.strip() #removes trailing spaces
1104                     if socket_text.isdigit() and int(socket_text)<9 and int(socket_text)>0:
1105                         socket_id = int(socket_text) - 1
1106
1107             elif entry.get("name") == 'family':
1108                 family = entry.text
1109                 count += 1
1110             elif entry.get("name") == 'manufacturer':
1111                 manufacturer = entry.text
1112                 count += 1
1113             elif entry.get("name") == 'version':
1114                 version = entry.text.strip()
1115                 count += 1
1116         if count != 3:
1117             return (False, 'Error. Not all expected fields could be found in processor')
1118
1119         #Create and fill processor structure
1120         if not_populated:
1121             continue  #avoid inconsistence of some machines where more socket detected than
1122         processor = ProcessorNode()
1123         (return_status, code) = processor.set(socket_id, family, manufacturer, version, capabilities, topology[socket_id])
1124         if not return_status:
1125             return (return_status, code)
1126         if code not in warning_text:
1127             warning_text += code
1128
1129         #Add processor to the processors dictionary
1130         processors[socket_id] = processor
1131
1132     return (True, warning_text)
1133
1134 def get_nic_information(ssh_conn, virsh_conn, nic_topology):
1135     warning_text=""
1136     #Get list of net devices
1137     net_devices = virsh_conn.listDevices('net',0)
1138     print virsh_conn.listDevices('net',0)
1139     for device in net_devices:
1140         try:
1141             #Get the XML descriptor of the device:
1142             net_XML = ElementTree.fromstring(virsh_conn.nodeDeviceLookupByName(device).XMLDesc(0))
1143             #print "net_XML:" , net_XML
1144             #obtain the parent
1145             parent = net_XML.find('parent')
1146             if parent == None:
1147                 print 'No parent was found in XML for device '+device
1148                 #Error. continue?-------------------------------------------------------------
1149                 continue
1150             if parent.text == 'computer':
1151                 continue
1152             if not parent.text.startswith('pci_'):
1153                 print device + ' parent is neither computer nor pci'
1154                 #Error. continue?-------------------------------------------------------------
1155                 continue
1156             interface = net_XML.find('capability/interface').text
1157             mac = net_XML.find('capability/address').text
1158
1159             #Get the pci XML
1160             pci_XML = ElementTree.fromstring(virsh_conn.nodeDeviceLookupByName(parent.text).XMLDesc(0))
1161             #print pci_XML
1162             #Get pci
1163             name = pci_XML.find('name').text.split('_')
1164             pci = name[1]+':'+name[2]+':'+name[3]+'.'+name[4]
1165
1166             #If slot == 0 it is a PF, otherwise it is a VF
1167             capability = pci_XML.find('capability')
1168             if capability.get('type') != 'pci':
1169                 print device + 'Capability is not of type pci in '+parent.text
1170                 #Error. continue?-------------------------------------------------------------
1171                 continue
1172             slot = capability.find('slot').text
1173             bus = capability.find('bus').text
1174             node_id = None
1175             numa_ = capability.find('numa')
1176             if numa_ != None:
1177                 node_id = numa_.get('node');
1178                 if node_id != None: node_id =int(node_id)
1179             if slot == None or bus == None:
1180                 print device + 'Bus and slot not detected in '+parent.text
1181                 #Error. continue?-------------------------------------------------------------
1182                 continue
1183             if slot != '0':
1184     #             print ElementTree.tostring(pci_XML)
1185                 virtual = True
1186                 capability_pf = capability.find('capability')
1187                 if capability_pf.get('type') != 'phys_function':
1188                     print 'physical_function not found in VF '+parent.text
1189                     #Error. continue?-------------------------------------------------------------
1190                     continue
1191                 PF_pci = capability_pf.find('address').attrib
1192                 PF_pci_text = PF_pci['domain'].split('x')[1]+':'+PF_pci['bus'].split('x')[1]+':'+PF_pci['slot'].split('x')[1]+'.'+PF_pci['function'].split('x')[1]
1193
1194             else:
1195                 virtual = False
1196
1197             #Obtain node for the port
1198             if node_id == None:
1199                 node_id = int(bus)>>6
1200             #print "node_id:", node_id
1201
1202             #Only for non virtual interfaces: Obtain speed and if link is detected (this must be done using ethtool)
1203             if not virtual:
1204                 command = 'sudo ethtool '+interface+' | grep -e Speed -e "Link detected"'
1205                 (_, stdout, stderr) = ssh_conn.exec_command(command)
1206                 error = stderr.read()
1207                 if len(error) >0:
1208                     print 'Error running '+command+'\n'+error
1209                     #Error. continue?-------------------------------------------------------------
1210                     continue
1211                 for line in stdout.readlines():
1212                     line = line.strip().rstrip('\n').split(': ')
1213                     if line[0] == 'Speed':
1214                         if line[1].endswith('Mb/s'):
1215                             speed = int(line[1].split('M')[0])*int(1e6)
1216                         elif line[1].endswith('Gb/s'):
1217                             speed = int(line[1].split('G')[0])*int(1e9)
1218                         elif line[1].endswith('Kb/s'):
1219                             speed = int(line[1].split('K')[0])*int(1e3)
1220                         else:
1221                             #the interface is listed but won't be used
1222                             speed = 0
1223                     elif line[0] == 'Link detected':
1224                         if line[1] == 'yes':
1225                             enabled = True
1226                         else:
1227                             enabled = False
1228                     else:
1229                         print 'Unnexpected output of command '+command+':'
1230                         print line
1231                         #Error. continue?-------------------------------------------------------------
1232                         continue
1233
1234             if not node_id in nic_topology:
1235                 nic_topology[node_id] = list()
1236                 #With this implementation we make the RAD with only one nic per node and this nic has all ports, TODO: change this by including parent information of PF
1237                 nic_topology[node_id].append(Nic())
1238
1239             #Load the appropriate nic
1240             nic = nic_topology[node_id][0]
1241
1242             #Create a new port and fill it
1243             port = Port()
1244             port.name = interface
1245             port.virtual = virtual
1246             port.speed = speed
1247             if virtual:
1248                 port.available_bw = 0
1249                 port.PF_pci_device_id = PF_pci_text
1250             else:
1251                 port.available_bw = speed
1252                 if speed == 0:
1253                     port.enabled = False
1254                 else:
1255                     port.enabled = enabled
1256
1257             port.eligible = virtual  #Only virtual ports are eligible
1258             port.mac = mac
1259             port.pci_device_id = pci
1260             port.pci_device_id_split = name[1:]
1261
1262             #Save the port information
1263             nic.add_port(port)
1264         except Exception,e:
1265             print 'Error: '+str(e)
1266
1267     #set in vitual ports if they are enabled
1268     for nic in nic_topology.itervalues():
1269         for port in nic[0].ports.itervalues():
1270 #             print port.pci_device_id
1271             if port.virtual:
1272                 enabled = nic[0].ports.get(port.PF_pci_device_id)
1273                 if enabled == None:
1274                     return(False, 'The PF '+port.PF_pci_device_id+' (VF '+port.pci_device_id+') is not present in ports dict')
1275                 #Only if the PF is enabled the VF can be enabled
1276                 if nic[0].ports[port.PF_pci_device_id].enabled:
1277                     port.enabled = True
1278                 else:
1279                     port.enabled = False
1280
1281     return (True, warning_text)
1282
1283 def get_nic_information_old(ssh_conn, nic_topology):
1284     command = 'lstopo-no-graphics --of xml'
1285     (_, stdout, stderr) = ssh_conn.exec_command(command)
1286     error = stderr.read()
1287     if len(error)>0:
1288         raise paramiko.ssh_exception.SSHException(command +' : '+ error)
1289     tree=ElementTree.fromstring(stdout.read())
1290     for target in tree.findall("object/object"):
1291         #Find numa nodes
1292         if target.get("type") != "NUMANode":
1293             continue
1294         node_id = int(target.get("os_index"))
1295         nic_topology[node_id] = list()
1296
1297         #find nics in numa node
1298         for entry in target.findall("object/object"):
1299             if entry.get("type") != 'Bridge':
1300                 continue
1301             nic_name = entry.get("name")
1302             model = None
1303             nic = Nic()
1304
1305             #find ports in nic
1306             for pcidev in entry.findall("object"):
1307                 if pcidev.get("type") != 'PCIDev':
1308                     continue
1309                 enabled = speed = mac = pci_busid = None
1310                 port = Port()
1311                 model = pcidev.get("name")
1312                 virtual = False
1313                 if 'Virtual' in model:
1314                     virtual = True
1315                 pci_busid = pcidev.get("pci_busid")
1316                 for osdev in pcidev.findall("object"):
1317                     name = osdev.get("name")
1318                     for info in osdev.findall("info"):
1319                         if info.get("name") != 'Address':
1320                             continue
1321                         mac = info.get("value")
1322                         #get the port speed and status
1323                         command = 'sudo ethtool '+name
1324                         (_, stdout, stderr) = ssh_conn.exec_command(command)
1325                         error = stderr.read()
1326                         if len(error)>0:
1327                             return (False, 'Error obtaining '+name+' information: '+error)
1328                         ethtool = stdout.read()
1329                         if '10000baseT/Full' in ethtool:
1330                             speed = 10e9
1331                         elif '1000baseT/Full' in ethtool:
1332                             speed = 1e9
1333                         elif '100baseT/Full' in ethtool:
1334                             speed = 100e6
1335                         elif '10baseT/Full' in ethtool:
1336                             speed = 10e6
1337                         else:
1338                             return (False, 'Speed not detected in '+name)
1339
1340                     enabled = False
1341                     if 'Link detected: yes' in ethtool:
1342                         enabled = True
1343
1344                     if speed != None and mac != None and pci_busid != None:
1345                         mac = mac.split(':')
1346                         pci_busid_split = re.split(':|\.', pci_busid)
1347                         #Fill the port information
1348                         port.set(name, virtual, enabled, speed, mac, pci_busid, pci_busid_split)
1349                         nic.add_port(port)
1350
1351             if len(nic.ports) > 0:
1352                 #Fill the nic model
1353                 if model != None:
1354                     nic.set_model(model)
1355                 else:
1356                     nic.set_model(nic_name)
1357
1358                 #Add it to the topology
1359                 nic_topology[node_id].append(nic)
1360
1361     return (True, "")
1362
1363 def get_os_information(ssh_conn, os):
1364     warning_text=""
1365 #    command = 'lsb_release -a'
1366 #    (stdin, stdout, stderr) = ssh_conn.exec_command(command)
1367 #    cont = 0
1368 #    for line in stdout.readlines():
1369 #        line_split = re.split('\t| *', line.rstrip('\n'))
1370 #        if line_split[0] == 'Distributor' and line_split[1] == 'ID:':
1371 #            distributor = line_split[2]
1372 #            cont += 1
1373 #        elif line_split[0] == 'Release:':
1374 #            release = line_split[1]
1375 #            cont += 1
1376 #        elif line_split[0] == 'Codename:':
1377 #            codename = line_split[1]
1378 #            cont += 1
1379 #    if cont != 3:
1380 #        return (False, 'It was not possible to obtain the OS id')
1381 #    id_ = distributor+'-'+release+'-'+codename
1382
1383
1384     command = 'cat /etc/redhat-release'
1385     (_, stdout, _) = ssh_conn.exec_command(command)
1386     id_text= stdout.read()
1387     if len(id_text)==0:
1388         #try with Ubuntu
1389         command = 'lsb_release -d -s'
1390         (_, stdout, _) = ssh_conn.exec_command(command)
1391         id_text= stdout.read()
1392     if len(id_text)==0:
1393         raise paramiko.ssh_exception.SSHException("Can not determinte release neither with 'lsb_release' nor with 'cat /etc/redhat-release'")
1394     id_ = id_text.rstrip('\n')
1395
1396     command = 'uname -o'
1397     (_, stdout, stderr) = ssh_conn.exec_command(command)
1398     error = stderr.read()
1399     if len(error)>0:
1400         raise paramiko.ssh_exception.SSHException(command +' : '+ error)
1401     type_ = stdout.read().rstrip('\n')
1402
1403     command = 'uname -i'
1404     (_, stdout, stderr) = ssh_conn.exec_command(command)
1405     error = stderr.read()
1406     if len(error)>0:
1407         raise paramiko.ssh_exception.SSHException(command +' : '+ error)
1408     bit_architecture = stdout.read().rstrip('\n')
1409
1410     (return_status, code) = os.set(id_, type_, bit_architecture)
1411     if not return_status:
1412         return (return_status, code)
1413     warning_text += code
1414     return (True, warning_text)
1415
1416 def get_hypervisor_information(virsh_conn, hypervisor):
1417     type_ = virsh_conn.getType().rstrip('\n')
1418     version = virsh_conn.getVersion()
1419     lib_version = virsh_conn.getLibVersion()
1420
1421     domains = list()
1422     tree=ElementTree.fromstring(virsh_conn.getCapabilities())
1423     for target in tree.findall("guest"):
1424         os_type = target.find("os_type").text
1425         #We only allow full virtualization
1426         if os_type != 'hvm':
1427             continue
1428         wordsize = int(target.find('arch/wordsize').text)
1429         if wordsize == 64:
1430             for domain in target.findall("arch/domain"):
1431                 domains.append(domain.get("type"))
1432
1433     (return_status, code) = hypervisor.set(type_, version, lib_version, domains)
1434     if not return_status:
1435         return (return_status, code)
1436     return (True, code)
1437
1438 class RADavailableResourcesClass(RADclass):
1439     def __init__(self, resources):
1440         """Copy resources from the RADclass (server resources not taking into account resources used by VMs"""
1441         #New
1442         self.reserved = dict()          #Dictionary of reserved resources for a server. Key are VNFC names and values RADreservedResources
1443         self.cores_consumption = None   #Dictionary of cpu consumption. Key is the cpu and the value is
1444
1445         self.machine = resources.machine
1446         self.user = resources.user
1447         self.password = resources.password
1448         self.name = resources.name
1449         self.nr_processors = resources.nr_processors
1450         self.processor_family = resources.processor_family
1451         self.processor_manufacturer = resources.processor_manufacturer
1452         self.processor_version = resources.processor_version
1453         self.processor_features = resources.processor_features
1454         self.memory_type = resources.memory_type
1455         self.memory_freq = resources.memory_freq
1456         self.memory_nr_channels = resources.memory_nr_channels
1457         self.memory_size = resources.memory_size
1458         self.memory_hugepage_sz = resources.memory_hugepage_sz
1459         self.hypervisor = Hypervisor()
1460         self.hypervisor.assign(resources.hypervisor)
1461         self.os = OpSys()
1462         self.os.assign(resources.os)
1463         self.nodes = dict()
1464         for node_k, node_v in resources.nodes.iteritems():
1465             self.nodes[node_k] = Node()
1466             self.nodes[node_k].assign(node_v)
1467         return
1468
1469     def _get_cores_consumption_warnings(self):
1470         """Returns list of warning strings in case warnings are generated.
1471         In case no warnings are generated the return value will be an empty list"""
1472         warnings = list()
1473         #Get the cores consumption
1474         (return_status, code) = get_ssh_connection(self.machine, self.user, self.password)
1475         if not return_status:
1476             return (return_status, code)
1477         ssh_conn = code
1478         command = 'mpstat -P ALL 1 1 | grep Average | egrep -v CPU\|all'
1479         (_, stdout, stderr) = ssh_conn.exec_command(command)
1480         error = stderr.read()
1481         if len(error) > 0:
1482             return (False, error)
1483
1484         self.cores_consumption = dict()
1485         for line in stdout.readlines():
1486             cpu_usage_split = re.split('\t| *', line.rstrip('\n'))
1487             usage = 100 *(1 - float(cpu_usage_split[10]))
1488             if usage > 0:
1489                 self.cores_consumption[int(cpu_usage_split[1])] = usage
1490         ssh_conn.close()
1491         #Check if any core marked as available in the nodes has cpu_usage > 0
1492         for _, node_v in self.nodes.iteritems():
1493             cores = node_v.processor.eligible_cores
1494             for cpu in cores:
1495                 if len(cpu) > 1:
1496                     for core in cpu:
1497                         if core in self.cores_consumption:
1498                             warnings.append('Warning: Core '+str(core)+' is supposed to be idle but it is consuming '+str(self.cores_consumption[core])+'%')
1499                 else:
1500                     if cpu in self.cores_consumption:
1501                         warnings.append('Warning: Core '+str(core)+' is supposed to be idle but it is consuming '+str(self.cores_consumption[cpu])+'%')
1502
1503         return warnings
1504
1505     def reserved_to_text(self):
1506         text = str()
1507         for VNFC_name, VNFC_reserved in self.reserved.iteritems():
1508             text += '    VNFC: '+str(VNFC_name)+'\n'
1509             text += VNFC_reserved.to_text()
1510
1511         return text
1512
1513     def obtain_usage(self):
1514         resp = dict()
1515         #Iterate through nodes to get cores, eligible cores, memory and physical ports (save ports usage for next section)
1516         nodes = dict()
1517         ports_usage = dict()
1518         hugepage_size = dict()
1519         for node_k, node_v in self.nodes.iteritems():
1520             node = dict()
1521             ports_usage[node_k] = dict()
1522             eligible_cores = list()
1523             for pair in node_v.processor.eligible_cores:
1524                 if isinstance(pair, list):
1525                     for element in pair:
1526                         eligible_cores.append(element)
1527                 else:
1528                     eligible_cores.append(pair)
1529             node['cpus'] = {'cores':node_v.processor.cores,'eligible_cores':eligible_cores}
1530             node['memory'] = {'size':str(node_v.memory.node_size/(1024*1024*1024))+'GB','eligible':str(node_v.memory.eligible_memory/(1024*1024*1024))+'GB'}
1531             hugepage_size[node_k] = node_v.memory.hugepage_sz
1532
1533             ports = dict()
1534             for nic in node_v.nic_list:
1535                 for port in nic.ports.itervalues():
1536                     if port.enabled and not port.virtual:
1537                         ports[port.name] = {'speed':str(port.speed/1000000000)+'G'}
1538 #                         print '*************** ',port.name,'speed',port.speed
1539                         ports_usage[node_k][port.name] = 100 - int(100*float(port.available_bw)/float(port.speed))
1540             node['ports'] = ports
1541             nodes[node_k] = node
1542         resp['RAD'] = nodes
1543
1544         #Iterate through reserved section to get used cores, used memory and port usage
1545         cores = dict()
1546         memory = dict()
1547         #reserved_cores = list
1548         for node_k in self.nodes.iterkeys():
1549             if not node_k in cores:
1550                 cores[node_k] = list()
1551                 memory[node_k] = 0
1552             for _, reserved in self.reserved.iteritems():
1553                 if node_k in reserved.node_reserved_resources:
1554                     node_v = reserved.node_reserved_resources[node_k]
1555                     cores[node_k].extend(node_v.reserved_cores)
1556                     memory[node_k] += node_v.reserved_hugepage_nr * hugepage_size[node_k]
1557
1558         occupation = dict()
1559         for node_k in self.nodes.iterkeys():
1560             ports = dict()
1561             for name, usage in ports_usage[node_k].iteritems():
1562                 ports[name] = {'occupied':str(usage)+'%'}
1563 #             print '****************cores',cores
1564 #             print '****************memory',memory
1565             occupation[node_k] = {'cores':cores[node_k],'memory':str(memory[node_k]/(1024*1024*1024))+'GB','ports':ports}
1566         resp['occupation'] = occupation
1567
1568         return resp
1569
1570 class RADreservedResources():
1571     def __init__(self):
1572         self.node_reserved_resources = dict()      #dict. keys are the RAD nodes id, values are NodeReservedResources
1573         self.mgmt_interface_pci = None             #pci in the VNF for the management interface
1574         self.image = None                          #Path in remote machine of the VNFC image
1575
1576     def update(self,reserved):
1577         self.image = reserved.image
1578         self.mgmt_interface_pci = reserved.mgmt_interface_pci
1579         for k,v in reserved.node_reserved_resources.iteritems():
1580             if k in self.node_reserved_resources.keys():
1581                 return (False, 'Duplicated node entry '+str(k)+' in reserved resources')
1582             self.node_reserved_resources[k]=v
1583
1584         return (True, "")
1585
1586     def to_text(self):
1587         text = '        image: '+str(self.image)+'\n'
1588         for node_id, node_reserved in self.node_reserved_resources.iteritems():
1589             text += '        Node ID: '+str(node_id)+'\n'
1590             text += node_reserved.to_text()
1591         return text
1592
1593 class NodeReservedResources():
1594     def __init__(self):
1595     #     reserved_shared_cores = None      #list. List of all cores that the VNFC needs in shared mode  #TODO Not used
1596     #     reserved_memory = None            #Integer. Amount of KiB needed by the VNFC #TODO. Not used since hugepages are used
1597         self.reserved_cores = list()             #list. List of all cores that the VNFC uses
1598         self.reserved_hugepage_nr = 0            #Integer. Number of hugepages needed by the VNFC
1599         self.reserved_ports = dict()             #dict. The key is the physical port pci and the value the VNFC port description
1600         self.vlan_tags = dict()
1601         self.cpu_pinning = None
1602
1603     def to_text(self):
1604         text = '            cores: '+str(self.reserved_cores)+'\n'
1605         text += '            cpu_pinning: '+str(self.cpu_pinning)+'\n'
1606         text += '            hugepages_nr: '+str(self.reserved_hugepage_nr)+'\n'
1607         for port_pci, port_description in self.reserved_ports.iteritems():
1608             text += '            port: '+str(port_pci)+'\n'
1609             text += port_description.to_text()
1610         return text
1611
1612 #     def update(self,reserved):
1613 #         self.reserved_cores = list(reserved.reserved_cores)
1614 #         self.reserved_hugepage_nr = reserved.reserved_hugepage_nr
1615 #         self.reserved_ports = dict(reserved.reserved_ports)
1616 #         self.cpu_pinning = list(reserved.cpu_pinning)
1617
1618
1619