| # -*- coding: utf-8 -*- |
| import code |
| |
| ## |
| # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. |
| # This file is part of openmano |
| # All Rights Reserved. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| # not use this file except in compliance with the License. You may obtain |
| # a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations |
| # under the License. |
| # |
| # For those usages not covered by the Apache License, Version 2.0 please |
| # contact with: nfvlabs@tid.es |
| ## |
| |
| ''' |
| Implement the logic for obtaining compute nodes information |
| Resource Availability Descriptor |
| ''' |
| __author__="Pablo Montes" |
| |
| #TODO: remove warnings, remove unused things |
| |
| from definitionsClass import definitionsClass |
| from auxiliary_functions import get_ssh_connection |
| import libvirt |
| from xml.etree import ElementTree |
| import paramiko |
| import re |
| import yaml |
| |
| |
| def getCredentials(creds, data): |
| """Used as a backup for libvirt.openAuth in order to provide password that came with data, |
| not used by the moment |
| """ |
| print "RADclass:getCredentials", creds, data |
| for cred in creds: |
| print cred[1] + ": ", |
| if cred[0] == libvirt.VIR_CRED_AUTHNAME: |
| cred[4] = data |
| elif cred[0] == libvirt.VIR_CRED_PASSPHRASE: |
| cred[4] = data |
| else: |
| return -1 |
| return 0 |
| |
| class RADclass(): |
| def __init__(self): |
| self.name = None |
| self.machine = None |
| self.user = None |
| self.password = None |
| self.nodes = dict() #Dictionary of nodes. Keys are the node id, values are Node() elements |
| self.nr_processors = None #Integer. Number of processors in the system |
| self.processor_family = None #If all nodes have the same value equal them, otherwise keep as None |
| self.processor_manufacturer = None #If all nodes have the same value equal them, otherwise keep as None |
| self.processor_version = None #If all nodes have the same value equal them, otherwise keep as None |
| self.processor_features = None #If all nodes have the same value equal them, otherwise keep as None |
| self.memory_type = None #If all nodes have the same value equal them, otherwise keep as None |
| self.memory_freq = None #If all nodes have the same value equal them, otherwise keep as None |
| self.memory_nr_channels = None #If all nodes have the same value equal them, otherwise keep as None |
| self.memory_size = None #Integer. Sum of the memory in all nodes |
| self.memory_hugepage_sz = None |
| self.hypervisor = Hypervisor() #Hypervisor information |
| self.os = OpSys() #Operating system information |
| self.ports_list = list() #List containing all network ports in the node. This is used to avoid having defined multiple times the same port in the system |
| |
| |
| def obtain_RAD(self, user, password, machine): |
| """This function obtains the RAD information from the remote server. |
| It uses both a ssh and a libvirt connection. |
| It is desirable in future versions get rid of the ssh connection, but currently |
| libvirt does not provide all the needed information. |
| Returns (True, Warning) in case of success and (False, <error>) in case of error""" |
| warning_text="" |
| try: |
| #Get virsh and ssh connection |
| (return_status, code) = get_ssh_connection(machine, user, password) |
| if not return_status: |
| print 'RADclass.obtain_RAD() error:', code |
| return (return_status, code) |
| ssh_conn = code |
| |
| self.connection_IP = machine |
| #print "libvirt open pre" |
| virsh_conn=libvirt.open("qemu+ssh://"+user+'@'+machine+"/system") |
| #virsh_conn=libvirt.openAuth("qemu+ssh://"+user+'@'+machine+"/system", |
| # [[libvirt.VIR_CRED_AUTHNAME, libvirt.VIR_CRED_PASSPHRASE, libvirt.VIR_CRED_USERNAME], getCredentials, password], |
| # 0) |
| #print "libvirt open after" |
| |
| # #Set connection infomation |
| # (return_status, code) = self.set_connection_info(machine, user, password) |
| # if not return_status: |
| # return (return_status, 'Error in '+machine+': '+code) |
| |
| #Set server name |
| machine_name = get_hostname(virsh_conn) |
| (return_status, code) = self.set_name(machine_name) |
| if not return_status: |
| return (return_status, 'Error at self.set_name in '+machine+': '+code) |
| warning_text += code |
| |
| #Get the server processors information |
| processors = dict() |
| (return_status, code) = get_processor_information(ssh_conn, virsh_conn, processors) |
| if not return_status: |
| return (return_status, 'Error at get_processor_information in '+machine+': '+code) |
| warning_text += code |
| |
| #Get the server memory information |
| memory_nodes = dict() |
| (return_status, code) = get_memory_information(ssh_conn, virsh_conn, memory_nodes) |
| if not return_status: |
| return (return_status, 'Error at get_memory_information in '+machine+': '+code) |
| warning_text += code |
| |
| #Get nics information |
| nic_topology = dict() |
| # (return_status, code) = get_nic_information_old(ssh_conn, nic_topology) |
| (return_status, code) = get_nic_information(ssh_conn, virsh_conn, nic_topology) |
| if not return_status: |
| return (return_status, 'Error at get_nic_information in '+machine+': '+code) |
| warning_text += code |
| |
| #Pack each processor, memory node and nics in a node element |
| #and add the node to the RAD element |
| for socket_id, processor in processors.iteritems(): |
| node = Node() |
| if not socket_id in nic_topology: |
| nic_topology[socket_id] = list() |
| |
| (return_status, code) = node.set(processor, memory_nodes[socket_id], nic_topology[socket_id]) |
| # else: |
| # (return_status, code) = node.set(processor, memory_nodes[socket_id]) |
| if not return_status: |
| return (return_status, 'Error at node.set in '+machine+': '+code) |
| warning_text += code |
| (return_status, code) = self.insert_node(node) |
| if not return_status: |
| return (return_status, 'Error at self.insert_node in '+machine+': '+code) |
| if code not in warning_text: |
| warning_text += code |
| |
| #Fill os data |
| os = OpSys() |
| (return_status, code) = get_os_information(ssh_conn, os) |
| if not return_status: |
| return (return_status, 'Error at get_os_information in '+machine+': '+code) |
| warning_text += code |
| (return_status, code) = self.set_os(os) |
| if not return_status: |
| return (return_status, 'Error at self.set_os in '+machine+': '+code) |
| warning_text += code |
| |
| #Fill hypervisor data |
| hypervisor = Hypervisor() |
| (return_status, code) = get_hypervisor_information(virsh_conn, hypervisor) |
| if not return_status: |
| return (return_status, 'Error at get_hypervisor_information in '+machine+': '+code) |
| warning_text += code |
| (return_status, code) = self.set_hypervisor(hypervisor) |
| if not return_status: |
| return (return_status, 'Error at self.set_hypervisor in '+machine+': '+code) |
| warning_text += code |
| ssh_conn.close() |
| |
| return (True, warning_text) |
| except libvirt.libvirtError, e: |
| text = e.get_error_message() |
| print 'RADclass.obtain_RAD() exception:', text |
| return (False, text) |
| except paramiko.ssh_exception.SSHException, e: |
| text = e.args[0] |
| print "obtain_RAD ssh Exception:", text |
| return False, text |
| |
| def set_name(self,name): |
| """Sets the machine name. |
| Returns (True,Warning) in case of success and ('False',<error description>) in case of error""" |
| if not isinstance(name,str): |
| return (False, 'The variable \'name\' must be text') |
| self.name = name |
| return (True, "") |
| |
| def set_connection_info(self, machine, user, password): |
| """Sets the connection information. |
| Returns (True,Warning) in case of success and ('False',<error description>) in case of error""" |
| if not isinstance(machine,str): |
| return (False, 'The variable \'machine\' must be text') |
| if not isinstance(user,str): |
| return (False, 'The variable \'user\' must be text') |
| # if not isinstance(password,str): |
| # return (False, 'The variable \'password\' must be text') |
| (self.machine, self.user, self.password) = (machine, user, password) |
| return (True, "") |
| |
| def insert_node(self,node): |
| """Inserts a new node and updates class variables. |
| Returns (True,Warning) in case of success and ('False',<error description>) in case of error""" |
| if not isinstance(node,Node): |
| return (False, 'The variable \'node\' must be a Node element') |
| |
| if node.id_ in self.nodes: |
| return (False, 'The node is already present in the nodes list.') |
| |
| #Check if network ports have not been inserted previously as part of another node |
| for port_key in node.ports_list: |
| if port_key in self.ports_list: |
| return (False, 'Network port '+port_key+' defined multiple times in the system') |
| self.ports_list.append(port_key) |
| |
| #Insert the new node |
| self.nodes[node.id_] = node |
| |
| #update variables |
| self.update_variables() |
| |
| return (True, "") |
| |
| def update_variables(self): |
| """Updates class variables. |
| Returns (True,Warning) in case of success and ('False',<error description>) in case of error""" |
| warning_text="" |
| #The number of processors and nodes is the same |
| self.nr_processors = len(self.nodes) |
| |
| #If all processors are the same get the values. Otherwise keep them as none |
| prev_processor_family = prev_processor_manufacturer = prev_processor_version = prev_processor_features = None |
| different_processor_family = different_processor_manufacturer = different_processor_version = different_processor_features = False |
| for node in self.nodes.itervalues(): |
| (self.processor_family, self.processor_manufacturer, self.processor_version, self.processor_features) = node.get_processor_info() |
| if prev_processor_family != None and self.processor_family != prev_processor_family: |
| different_processor_family = True |
| if prev_processor_manufacturer != None and self.processor_manufacturer != prev_processor_manufacturer: |
| different_processor_manufacturer = True |
| if prev_processor_version != None and self.processor_version != prev_processor_version: |
| different_processor_version = True |
| if prev_processor_features != None and self.processor_features != prev_processor_features: |
| different_processor_features = True |
| (prev_processor_family, prev_processor_manufacturer, prev_processor_version, prev_processor_features) = (self.processor_family, self.processor_manufacturer, self.processor_version, self.processor_features) |
| |
| if different_processor_family: |
| self.processor_family = None |
| if different_processor_features: |
| self.processor_features = None |
| if different_processor_manufacturer: |
| self.processor_manufacturer = None |
| if different_processor_version: |
| self.processor_version = None |
| |
| #If all memory nodes are the same get the values. Otherwise keep them as none |
| #Sum the total memory |
| self.memory_size = 0 |
| different_memory_freq = different_memory_nr_channels = different_memory_type = different_memory_hugepage_sz = False |
| prev_memory_freq = prev_memory_nr_channels = prev_memory_type = prev_memory_hugepage_sz = None |
| for node in self.nodes.itervalues(): |
| (self.memory_freq, self.memory_nr_channels, self.memory_type, memory_size, self.memory_hugepage_sz) = node.get_memory_info() |
| self.memory_size += memory_size |
| if prev_memory_freq != None and self.memory_freq != prev_memory_freq: |
| different_memory_freq = True |
| if prev_memory_nr_channels != None and self.memory_nr_channels != prev_memory_nr_channels: |
| different_memory_nr_channels = True |
| if prev_memory_type != None and self.memory_type != prev_memory_type: |
| different_memory_type = True |
| if prev_memory_hugepage_sz != None and self.memory_hugepage_sz != prev_memory_hugepage_sz: |
| different_memory_hugepage_sz = True |
| (prev_memory_freq, prev_memory_nr_channels, prev_memory_type, prev_memory_hugepage_sz) = (self.memory_freq, self.memory_nr_channels, self.memory_type, self.memory_hugepage_sz) |
| |
| if different_memory_freq: |
| self.memory_freq = None |
| if different_memory_nr_channels: |
| self.memory_nr_channels = None |
| if different_memory_type: |
| self.memory_type = None |
| if different_memory_hugepage_sz: |
| warning_text += 'Detected different hugepages size in different sockets\n' |
| |
| return (True, warning_text) |
| |
| def set_hypervisor(self,hypervisor): |
| """Sets the hypervisor. |
| Returns (True,Warning) in case of success and ('False',<error description>) in case of error""" |
| if not isinstance(hypervisor,Hypervisor): |
| return (False, 'The variable \'hypervisor\' must be of class Hypervisor') |
| |
| self.hypervisor.assign(hypervisor) |
| return (True, "") |
| |
| def set_os(self,os): |
| """Sets the operating system. |
| Returns (True,Warning) in case of success and ('False',<error description>) in case of error""" |
| if not isinstance(os,OpSys): |
| return (False, 'The variable \'os\' must be of class OpSys') |
| |
| self.os.assign(os) |
| return (True, "") |
| |
| def to_text(self): |
| text= 'name: '+str(self.name)+'\n' |
| text+= 'processor:\n' |
| text+= ' nr_processors: '+str(self.nr_processors)+'\n' |
| text+= ' family: '+str(self.processor_family)+'\n' |
| text+= ' manufacturer: '+str(self.processor_manufacturer)+'\n' |
| text+= ' version: '+str(self.processor_version)+'\n' |
| text+= ' features: '+str(self.processor_features)+'\n' |
| text+= 'memory:\n' |
| text+= ' type: '+str(self.memory_type)+'\n' |
| text+= ' freq: '+str(self.memory_freq)+'\n' |
| text+= ' nr_channels: '+str(self.memory_nr_channels)+'\n' |
| text+= ' size: '+str(self.memory_size)+'\n' |
| text+= 'hypervisor:\n' |
| text+= self.hypervisor.to_text() |
| text+= 'os:\n' |
| text+= self.os.to_text() |
| text+= 'resource topology:\n' |
| text+= ' nr_nodes: '+ str(len(self.nodes))+'\n' |
| text+= ' nodes:\n' |
| for node_k, node_v in self.nodes.iteritems(): |
| text+= ' node'+str(node_k)+':\n' |
| text+= node_v.to_text() |
| return text |
| |
| def to_yaml(self): |
| return yaml.load(self.to_text()) |
| |
| class Node(): |
| def __init__(self): |
| self.id_ = None #Integer. Node id. Unique in the system |
| self.processor = ProcessorNode() #Information about the processor in the node |
| self.memory = MemoryNode() #Information about the memory in the node |
| self.nic_list = list() #List of Nic() containing information about the nics associated to the node |
| self.ports_list = list() #List containing all network ports in the node. This is used to avoid having defined multiple times the same port in the system |
| |
| def get_processor_info(self): |
| """Gets the processor information. Returns (processor_family, processor_manufacturer, processor_version, processor_features)""" |
| return self.processor.get_info() |
| |
| def get_memory_info(self): |
| """Gets the memory information. Returns (memory_freq, memory_nr_channels, memory_type, memory_size)""" |
| return self.memory.get_info() |
| |
| # def set(self, *args): |
| # """Sets the node information. Returns (True,Warning) in case of success and ('False',<error description>) in case of error""" |
| # if len(args)==2: |
| # processor = args[0] |
| # memory = args[1] |
| # nics = False |
| # elif len(args)==3: |
| # processor = args[0] |
| # memory = args[1] |
| # nic_list = args[2] |
| # nics = True |
| # else: |
| # return (False, 'Wrong number of elements calling Node().set()') |
| |
| def set(self, processor, memory, nic_list): |
| (status, return_code) = self.processor.assign(processor) |
| if not status: |
| return (status, return_code) |
| |
| self.id_ = processor.id_ |
| |
| (status, return_code) = self.memory.assign(memory) |
| if not status: |
| return (status, return_code) |
| |
| # if nics: |
| for nic in nic_list: |
| if not isinstance(nic,Nic): |
| return (False, 'The nics must be of type Nic') |
| self.nic_list.append(nic) |
| for port_key in nic.ports.iterkeys(): |
| if port_key in self.ports_list: |
| return (False, 'Network port '+port_key+'defined multiple times in the same node') |
| self.ports_list.append(port_key) |
| |
| return (True,"") |
| |
| def assign(self, node): |
| """Sets the node information. |
| Returns (True,Warning) in case of success and ('False',<error description>) in case of error""" |
| warning_text="" |
| processor = node.processor |
| memory = node.memory |
| nic_list = node.nic_list |
| (status, return_code) = self.processor.assign(processor) |
| if not status: |
| return (status, return_code) |
| |
| self.id_ = processor.id_ |
| |
| (status, return_code) = self.memory.assign(memory) |
| if not status: |
| return (status, return_code) |
| warning_text += code |
| |
| for nic in nic_list: |
| if not isinstance(nic,Nic): |
| return (False, 'The nics must be of type Nic') |
| self.nic_list.append(nic) |
| for port_key in nic.ports.iterkeys(): |
| if port_key in self.ports_list: |
| return (False, 'Network port '+port_key+'defined multiple times in the same node') |
| self.ports_list.append(port_key) |
| |
| return (True,warning_text) |
| |
| def to_text(self): |
| text= ' id: '+str(self.id_)+'\n' |
| text+= ' cpu:\n' |
| text += self.processor.to_text() |
| text+= ' memory:\n' |
| text += self.memory.to_text() |
| if len(self.nic_list) > 0: |
| text+= ' nics:\n' |
| nic_index = 0 |
| for nic in self.nic_list: |
| text+= ' nic '+str(nic_index)+':\n' |
| text += nic.to_text() |
| nic_index += 1 |
| return text |
| |
| class ProcessorNode(): |
| #Definition of the possible values of processor variables |
| possible_features = definitionsClass.processor_possible_features |
| possible_manufacturers = definitionsClass.processor_possible_manufacturers |
| possible_families = definitionsClass.processor_possible_families |
| possible_versions = definitionsClass.processor_possible_versions |
| |
| def __init__(self): |
| self.id_ = None #Integer. Numeric identifier of the socket |
| self.family = None #Text. Family name of the processor |
| self.manufacturer = None #Text. Manufacturer of the processor |
| self.version = None #Text. Model version of the processor |
| self.features = list() #list. List of features offered by the processor |
| self.cores = list() #list. List of cores in the processor. In case of hyperthreading the coupled cores are expressed as [a,b] |
| self.eligible_cores = list()#list. List of cores that can be used |
| #self.decicated_cores |
| #self.shared_cores -> this should also contain information to know if cores are being used |
| |
| def assign(self, processor): |
| """Sets the processor information. |
| Returns (True,Warning) in case of success and ('False',<error description>) in case of error""" |
| if not isinstance(processor,ProcessorNode): |
| return (False, 'The variable \'processor\' must be of class ProcessorNode') |
| |
| self.id_ = processor.id_ |
| self.family = processor.family |
| self.manufacturer = processor.manufacturer |
| self.version = processor.version |
| self.features = processor.features |
| self.cores = processor.cores |
| self.eligible_cores = processor.eligible_cores |
| |
| return (True, "") |
| |
| def set(self, id_, family, manufacturer, version, features, cores): |
| """Sets the processor information. |
| Returns (True,Warning) in case of success and ('False',<error description>) in case of error""" |
| warning_text = "" |
| |
| if not isinstance(id_,int): |
| return (False, 'The processor id_ must be of type int') |
| if not isinstance(family,str): |
| return (False, 'The processor family must be of type str') |
| if not isinstance(manufacturer,str): |
| return (False, 'The processor manufacturer must be of type str') |
| if not isinstance(version,str): |
| return (False, 'The processor version must be of type str') |
| if not isinstance(features,list): |
| return (False, 'The processor features must be of type list') |
| if not isinstance(cores,list): |
| return (False, 'The processor cores must be of type list') |
| |
| (self.id_, self.family, self.manufacturer, self.version) = (id_, family, manufacturer, version) |
| |
| if not manufacturer in self.possible_manufacturers: |
| warning_text += "processor manufacturer '%s' not among: %s\n" %(manufacturer, str(self.possible_manufacturers)) |
| if not family in self.possible_families: |
| warning_text += "family '%s' not among: %s\n" % (family, str(self.possible_families)) |
| # if not version in self.possible_versions: |
| # warning_text += 'The version %s is not one of these: %s\n' % (version, str(self.possible_versions)) |
| |
| for feature in features: |
| if not feature in self.possible_features: |
| warning_text += "processor feature '%s' not among: %s\n" % (feature, str(self.possible_versions)) |
| self.features.append(feature) |
| |
| for iterator in sorted(cores): |
| if not isinstance(iterator,list) or not all(isinstance(x, int) for x in iterator): |
| return (False, 'The cores list must be in the form of [[a,b],[c,d],...] where a,b,c,d are of type int') |
| self.cores.append(iterator) |
| |
| self.set_eligible_cores() |
| |
| return (True,warning_text) |
| |
| def set_eligible_cores(self): |
| """Set the default eligible cores, this is all cores non used by the host operating system""" |
| not_first = False |
| for iterator in self.cores: |
| if not_first: |
| self.eligible_cores.append(iterator) |
| else: |
| not_first = True |
| return |
| |
| def get_info(self): |
| """Returns processor parameters (self.family, self.manufacturer, self.version, self.features)""" |
| return (self.family, self.manufacturer, self.version, self.features) |
| |
| def to_text(self): |
| text= ' id: '+str(self.id_)+'\n' |
| text+= ' family: '+self.family+'\n' |
| text+= ' manufacturer: '+self.manufacturer+'\n' |
| text+= ' version: '+self.version+'\n' |
| text+= ' features: '+str(self.features)+'\n' |
| text+= ' cores: '+str(self.cores)+'\n' |
| text+= ' eligible_cores: '+str(self.eligible_cores)+'\n' |
| return text |
| |
| class MemoryNode(): |
| def __init__(self): |
| self.modules = list() #List of MemoryModule(). List of all modules installed in the node |
| self.nr_channels = None #Integer. Number of modules installed in the node |
| self.node_size = None #Integer. Total size in KiB of memory installed in the node |
| self.eligible_memory = None #Integer. Size in KiB of eligible memory in the node |
| self.hugepage_sz = None #Integer. Size in KiB of hugepages |
| self.hugepage_nr = None #Integer. Number of hugepages allocated in the module |
| self.eligible_hugepage_nr = None #Integer. Number of eligible hugepages in the node |
| self.type_ = None #Text. Type of memory modules. If modules have a different value keep it as None |
| self.freq = None #Integer. Frequency of the modules in MHz. If modules have a different value keep it as None |
| self.module_size = None #Integer. Size of the modules in KiB. If modules have a different value keep it as None |
| self.form_factor = None #Text. Form factor of the modules. If modules have a different value keep it as None |
| |
| def assign(self, memory_node): |
| return self.set(memory_node.modules, memory_node.hugepage_sz, memory_node.hugepage_nr) |
| |
| def set(self, modules, hugepage_sz, hugepage_nr): |
| """Set the memory node information. hugepage_sz must be expressed in KiB. |
| Returns (True,Warning) in case of success and ('False',<error description>) in case of error""" |
| if not isinstance(modules, list): |
| return (False, 'The modules must be a list of elements of class MemoryModule') |
| if not isinstance(hugepage_sz,int): |
| return (False, 'The hugepage_sz variable must be an int expressing the size in KiB') |
| if not isinstance(hugepage_nr,int): |
| return (False, 'The hugepage_nr variable must be of type int') |
| |
| (self.hugepage_sz, self.hugepage_nr) = (hugepage_sz, hugepage_nr) |
| self.node_size = self.nr_channels = 0 |
| |
| different_type = different_freq = different_module_size = different_form_factor = False |
| prev_type = prev_freq = prev_module_size = prev_form_factor = None |
| for iterator in modules: |
| if not isinstance(iterator,MemoryModule): |
| return (False, 'The modules must be a list of elements of class MemoryModule') |
| self.modules.append(iterator) |
| (self.type_, self.freq, self.module_size, self.form_factor) = (iterator.type_, iterator.freq, iterator.size, iterator.form_factor) |
| self.node_size += self.module_size |
| self.nr_channels += 1 |
| if prev_type != None and prev_type != self.type_: |
| different_type = True |
| if prev_freq != None and prev_freq != self.freq: |
| different_freq = True |
| if prev_module_size != None and prev_module_size != self.module_size: |
| different_module_size = True |
| if prev_form_factor != None and prev_form_factor != self.form_factor: |
| different_form_factor = True |
| (prev_type, prev_freq, prev_module_size, prev_form_factor) = (self.type_, self.freq, self.module_size, self.form_factor) |
| |
| if different_type: |
| self.type_ = None |
| if different_freq: |
| self.freq = None |
| if different_module_size: |
| self.module_size = None |
| if different_form_factor: |
| self.form_factor = None |
| |
| (return_value, error_code) = self.set_eligible_memory() |
| if not return_value: |
| return (return_value, error_code) |
| |
| return (True, "") |
| |
| def set_eligible_memory(self): |
| """Sets the default eligible_memory and eligible_hugepage_nr. This is all memory but 2GiB and all hugepages""" |
| self.eligible_memory = self.node_size - 2*1024*1024 |
| if self.eligible_memory < 0: |
| return (False, "There is less than 2GiB of memory in the module") |
| |
| self.eligible_hugepage_nr = self.hugepage_nr |
| return (True,"") |
| |
| def get_info(self): |
| """Return memory information (self.freq, self.nr_channels, self.type_, self.node_size)""" |
| return (self.freq, self.nr_channels, self.type_, self.node_size, self.hugepage_sz) |
| |
| def to_text(self): |
| text= ' node_size: '+str(self.node_size)+'\n' |
| text+= ' nr_channels: '+str(self.nr_channels)+'\n' |
| text+= ' eligible_memory: '+str(self.eligible_memory)+'\n' |
| text+= ' hugepage_sz: '+str(self.hugepage_sz)+'\n' |
| text+= ' hugepage_nr: '+str(self.hugepage_nr)+'\n' |
| text+= ' eligible_hugepage_nr: '+str(self.eligible_hugepage_nr)+'\n' |
| text+= ' type: '+self.type_+'\n' |
| text+= ' freq: '+str(self.freq)+'\n' |
| text+= ' module_size: '+str(self.module_size)+'\n' |
| text+= ' form_factor: '+self.form_factor+'\n' |
| text+= ' modules details:\n' |
| for module in self.modules: |
| text += module.to_text() |
| return text |
| |
| class MemoryModule(): |
| #Definition of the possible values of module variables |
| possible_types = definitionsClass.memory_possible_types |
| possible_form_factors = definitionsClass.memory_possible_form_factors |
| |
| def __init__(self): |
| self.locator = None #Text. Name of the memory module |
| self.type_ = None #Text. Type of memory module |
| self.freq = None #Integer. Frequency of the module in MHz |
| self.size = None #Integer. Size of the module in KiB |
| self.form_factor = None #Text. Form factor of the module |
| |
| def set(self, locator, type_, freq, size, form_factor): |
| """Sets the memory module information. |
| Frequency must be expressed in MHz and size in KiB. |
| Returns (True,Warning) in case of success and ('False',<error description>) in case of error""" |
| warning_text="" |
| if not isinstance(locator, str): |
| return (False, "The type of the variable locator must be str") |
| if not isinstance(type_, str): |
| return (False, "The type of the variable type_ must be str") |
| if not isinstance(form_factor, str): |
| return (False, "The type of the variable form_factor must be str") |
| if not isinstance(freq, int): |
| return (False, "The type of the variable freq must be int") |
| if not isinstance(size, int): |
| return (False, "The type of the variable size must be int") |
| |
| if not form_factor in self.possible_form_factors: |
| warning_text += "memory form_factor '%s' not among: %s\n" %(form_factor, str(self.possible_form_factors)) |
| if not type_ in self.possible_types: |
| warning_text += "memory type '%s' not among: %s\n" %(type_, str(self.possible_types)) |
| |
| (self.locator, self.type_, self.freq, self.size, self.form_factor) = (locator, type_, freq, size, form_factor) |
| return (True, warning_text) |
| |
| def to_text(self): |
| text= ' '+self.locator+':\n' |
| text+= ' type: '+self.type_+'\n' |
| text+= ' freq: '+str(self.freq)+'\n' |
| text+= ' size: '+str(self.size)+'\n' |
| text+= ' form factor: '+self.form_factor+'\n' |
| return text |
| |
| class Nic(): |
| def __init__(self): |
| self.model = None #Text. Model of the nic |
| self.ports = dict() #Dictionary of ports. Keys are the port name, value are Port() elements |
| |
| def set_model(self, model): |
| """Sets the model of the nic. Returns (True,Warning) in case of success and ('False',<error description>) in case of error""" |
| if not isinstance(model,str): |
| return (False, 'The \'model\' must be of type str') |
| |
| self.model = model |
| return (True, "") |
| |
| def add_port(self, port): |
| """Adds a port to the nic. Returns (True,Warning) in case of success and ('False',<error description>) in case of error""" |
| if not isinstance(port,Port): |
| return (False, 'The \'port\' must be of class Port') |
| |
| # port_id = str(port.pci_device_id[0])+':'+str(port.pci_device_id[1])+':'+str(port.pci_device_id[2])+'.'+str(port.pci_device_id[3]) |
| #CHANGED |
| # port_id = port.name |
| port_id = port.pci_device_id |
| #CHANGED END |
| if port_id in self.ports: |
| return (False, 'The \'port\' '+port.pci_device_id+' is duplicated in the nic') |
| # return (False, 'The \'port\' is duplicated in the nic') |
| |
| self.ports[port_id] = port |
| return (True, "") |
| |
| def to_text(self): |
| text= ' model: '+ str(self.model)+'\n' |
| text+= ' ports: '+'\n' |
| for key,port in self.ports.iteritems(): |
| text+= ' "'+key+'":'+'\n' |
| text += port.to_text() |
| return text |
| |
| class Port(): |
| def __init__(self): |
| self.name = None #Text. Port name |
| self.virtual = None #Boolean. States if the port is a virtual function |
| self.enabled = None #Boolean. States if the port is enabled |
| self.eligible = None #Boolean. States if the port is eligible |
| self.speed = None #Integer. Indicates the speed in Mbps |
| self.available_bw = None #Integer. BW in Mbps that is available. |
| self.mac = None #list. Indicates the mac address of the port as a list in format ['XX','XX','XX','XX','XX','XX'] |
| self.pci_device_id_split = None #list. Indicates the pci address of the port as a list in format ['XXXX','XX','XX','X'] |
| self.pci_device_id = None |
| self.PF_pci_device_id = None |
| |
| # def set(self, name, virtual, enabled, speed, mac, pci_device_id, pci_device_id_split): |
| # """Sets the port information. The variable speed indicates the speed in Mbps. Returns (True,Warning) in case of success and ('False',<error description>) in case of error""" |
| # if not isinstance(name,str): |
| # return (False, 'The variable \'name\' must be of type str') |
| # if not isinstance(virtual,bool): |
| # return (False, 'The variable \'virtual\' must be of type bool') |
| # if not isinstance(enabled,bool): |
| # return (False, 'The variable \'enabled\' must be of type bool') |
| # if not isinstance(enabled,bool): |
| # return (speed, 'The variable \'speed\' must be of type int') |
| # if not isinstance(mac, list) and not isinstance(mac,NoneType): |
| # return (False, 'The variable \'enabled\' must be of type list indicating the mac address in format [\'XXXX\',\'XX\',\'XX\',\'X\'] or NoneType') |
| # if not isinstance(pci_device_id_split, list) or len(pci_device_id_split) != 4: |
| # return (False, 'The variable \'pci_device_id_split\' must be of type list, indicating the pci address in format [\'XX\',\'XX\',\'XX\',\'XX\',\'XX\',\'XX\']') |
| # |
| # expected_len = [4,2,2,1] |
| # index = 0 |
| # for iterator in pci_device_id_split: |
| # if not isinstance(iterator,str) or not iterator.isdigit() or len(iterator) != expected_len[index]: |
| # return (False, 'The variable \'pci_device_id_split\' must be of type list, indicating the pci address in format [\'XX\',\'XX\',\'XX\',\'XX\',\'XX\',\'XX\']') |
| # index += 1 |
| # |
| # if not isinstance(mac,NoneType): |
| # for iterator in mac: |
| # if not isinstance(iterator,str) or not iterator.isalnum() or len(iterator) != 2: |
| # return (False, 'The variable \'enabled\' must be of type list indicating the mac address in format [\'XXXX\',\'XX\',\'XX\',\'X\'] or NoneType') |
| # |
| # #By default only virtual ports are eligible |
| # # (self.name, self.virtual, self.enabled, self.eligible, self.available_bw, self.speed, self.mac, self.pci_device_id, self.pci_device_id_split) = (name, virtual, enabled, virtual, speed, speed, mac, pci_device_id, pci_device_id_split) |
| # (self.name, self.virtual, self.enabled, self.eligible, self.available_bw, self.mac, self.pci_device_id, self.pci_device_id_split) = (name, virtual, enabled, virtual, speed, mac, pci_device_id, pci_device_id_split) |
| |
| def to_text(self): |
| text= ' pci: "'+ str(self.pci_device_id)+'"\n' |
| text+= ' virtual: '+ str(self.virtual)+'\n' |
| if self.virtual: |
| text+= ' PF_pci_id: "'+self.PF_pci_device_id+'"\n' |
| text+= ' eligible: '+ str(self.eligible)+'\n' |
| text+= ' enabled: '+str(self.enabled)+'\n' |
| text+= ' speed: '+ str(self.speed)+'\n' |
| text+= ' available bw: '+ str(self.available_bw)+'\n' |
| text+= ' mac: '+ str(self.mac)+'\n' |
| text+= ' source_name: '+ str(self.name)+'\n' |
| return text |
| |
| class Hypervisor(): |
| #Definition of the possible values of hypervisor variables |
| possible_types = definitionsClass.hypervisor_possible_types |
| possible_domain_types = definitionsClass.hypervisor_possible_domain_types |
| |
| def __init__(self): |
| self.type_ = None #Text. Hypervisor type_ |
| self.version = None #int. Hypervisor version |
| self.lib_version = None #int. Libvirt version used to compile hypervisor |
| self.domains = list() #list. List of all the available domains |
| |
| def set(self, hypervisor, version, lib_version, domains): |
| warning_text="" |
| if not isinstance(hypervisor,str): |
| return (False, 'The variable type_ must be of type str') |
| if not isinstance(version,int): |
| return (False, 'The variable version must be of type int') |
| if not isinstance(lib_version,int): |
| return (False, 'The library version must be of type int') |
| if not isinstance(domains,list): |
| return (False, 'Domains must be a list of the possible domains as str') |
| |
| if not hypervisor in self.possible_types: |
| warning_text += "Hyperpivor '%s' not among: %s\n" % (hypervisor, str(self.possible_types)) |
| |
| valid_domain_found = False |
| for domain in domains: |
| if not isinstance(domain,str): |
| return (False, 'Domains must be a list of the possible domains as str') |
| if domain in self.possible_domain_types: |
| valid_domain_found = True |
| self.domains.append(domain) |
| |
| if not valid_domain_found: |
| warning_text += 'No valid domain found among: %s\n' % str(self.possible_domain_types) |
| |
| |
| (self.version, self.lib_version, self.type_) = (version, lib_version, hypervisor) |
| return (True, warning_text) |
| |
| def assign(self, hypervisor): |
| (self.version, self.lib_version, self.type_) = (hypervisor.version, hypervisor.lib_version, hypervisor.type_) |
| for domain in hypervisor.domains: |
| self.domains.append(domain) |
| return |
| |
| def to_text(self): |
| text= ' type: '+self.type_+'\n' |
| text+= ' version: '+str(self.version)+'\n' |
| text+= ' libvirt version: '+ str(self.lib_version)+'\n' |
| text+= ' domains: '+str(self.domains)+'\n' |
| return text |
| |
| class OpSys(): |
| #Definition of the possible values of os variables |
| possible_id = definitionsClass.os_possible_id |
| possible_types = definitionsClass.os_possible_types |
| possible_architectures = definitionsClass.os_possible_architectures |
| |
| def __init__(self): |
| self.id_ = None #Text. Identifier of the OS. Formed by <Distibutor ID>-<Release>-<Codename>. In linux this can be obtained using lsb_release -a |
| self.type_ = None #Text. Type of operating system |
| self.bit_architecture = None #Integer. Architecture |
| |
| def set(self, id_, type_, bit_architecture): |
| warning_text="" |
| if not isinstance(type_,str): |
| return (False, 'The variable type_ must be of type str') |
| if not isinstance(id_,str): |
| return (False, 'The variable id_ must be of type str') |
| if not isinstance(bit_architecture,str): |
| return (False, 'The variable bit_architecture must be of type str') |
| |
| if not type_ in self.possible_types: |
| warning_text += "os type '%s' not among: %s\n" %(type_, str(self.possible_types)) |
| if not id_ in self.possible_id: |
| warning_text += "os release '%s' not among: %s\n" %(id_, str(self.possible_id)) |
| if not bit_architecture in self.possible_architectures: |
| warning_text += "os bit_architecture '%s' not among: %s\n" % (bit_architecture, str(self.possible_architectures)) |
| |
| (self.id_, self.type_, self.bit_architecture) = (id_, type_, bit_architecture) |
| return (True, warning_text) |
| |
| def assign(self,os): |
| (self.id_, self.type_, self.bit_architecture) = (os.id_, os.type_, os.bit_architecture) |
| return |
| |
| def to_text(self): |
| text= ' id: '+self.id_+'\n' |
| text+= ' type: '+self.type_+'\n' |
| text+= ' bit_architecture: '+self.bit_architecture+'\n' |
| return text |
| |
| def get_hostname(virsh_conn): |
| return virsh_conn.getHostname().rstrip('\n') |
| |
| def get_hugepage_size(ssh_conn): |
| command = 'sudo hugeadm --page-sizes' |
| # command = 'hugeadm --page-sizes-all' |
| (_, stdout, stderr) = ssh_conn.exec_command(command) |
| error = stderr.read() |
| if len(error)>0: |
| raise paramiko.ssh_exception.SSHException(command +' : '+ error) |
| mem=stdout.read() |
| if mem=="": |
| return 0 |
| return int(mem) |
| |
| def get_hugepage_nr(ssh_conn,hugepage_sz, node_id): |
| command = 'cat /sys/devices/system/node/node'+str(node_id)+'/hugepages/hugepages-'+str(hugepage_sz/1024)+'kB/nr_hugepages' |
| (_, stdout, _) = ssh_conn.exec_command(command) |
| #print command, |
| #text = stdout.read() |
| #print "'"+text+"'" |
| #return int(text) |
| |
| try: |
| value=int(stdout.read()) |
| except: |
| value=0 |
| return value |
| |
| def get_memory_information(ssh_conn, virsh_conn, memory_nodes): |
| warning_text="" |
| tree=ElementTree.fromstring(virsh_conn.getSysinfo(0)) |
| memory_dict = dict() |
| node_id = 0 #TODO revise. Added for allowing VM as compute hosts |
| for target in tree.findall("memory_device"): |
| locator_f = size_f = freq_f = type_f = formfactor_f = False |
| locator_f = True #TODO revise. Added for allowing VM as compute hosts |
| module_form_factor = "" |
| for entry in target.findall("entry"): |
| if entry.get("name") == 'size': |
| size_f = True |
| size_split = entry.text.split(' ') |
| if size_split[1] == 'MB': |
| module_size = int(size_split[0]) * 1024 * 1024 |
| elif size_split[1] == 'GB': |
| module_size = int(size_split[0]) * 1024 * 1024 * 1024 |
| elif size_split[1] == 'KB': |
| module_size = int(size_split[0]) * 1024 |
| else: |
| module_size = int(size_split[0]) |
| |
| elif entry.get("name") == 'speed': |
| freq_f = True |
| freq_split = entry.text.split(' ') |
| if freq_split[1] == 'MHz': |
| module_freq = int(freq_split[0]) * 1024 * 1024 |
| elif freq_split[1] == 'GHz': |
| module_freq = int(freq_split[0]) * 1024 * 1024 * 1024 |
| elif freq_split[1] == 'KHz': |
| module_freq = int(freq_split[0]) * 1024 |
| |
| elif entry.get("name") == 'type': |
| type_f = True |
| module_type = entry.text |
| |
| elif entry.get("name") == 'form_factor': |
| formfactor_f = True |
| module_form_factor = entry.text |
| #TODO revise. Commented for allowing VM as compute hosts |
| # elif entry.get("name") == 'locator' and not locator_f: |
| # # other case, it is obtained by bank_locator that we give priority to |
| # locator = entry.text |
| # pos = locator.find(module_form_factor) |
| # if module_form_factor == locator[0:len(module_form_factor) ]: |
| # pos = len(module_form_factor) +1 |
| # else: |
| # pos = 0 |
| # if locator[pos] in "ABCDEFGH": |
| # locator_f = True |
| # node_id = ord(locator[pos])-ord('A') |
| # #print entry.text, node_id |
| # |
| # elif entry.get("name") == 'bank_locator': |
| # locator = entry.text |
| # pos = locator.find("NODE ") |
| # if pos >= 0 and len(locator)>pos+5: |
| # if locator[pos+5] in ("01234567"): #len("NODE ") is 5 |
| # node_id = int(locator[pos+5]) |
| # locator_f = True |
| # |
| |
| #When all module fields have been found add a new module to the list |
| if locator_f and size_f and freq_f and type_f and formfactor_f: |
| #If the memory node has not yet been created create it |
| if node_id not in memory_dict: |
| memory_dict[node_id] = [] |
| |
| #Add a new module to the memory node |
| module = MemoryModule() |
| #TODO revise. Changed for allowing VM as compute hosts |
| (return_status, code) = module.set('NODE %d' % node_id, module_type, module_freq, module_size, module_form_factor) |
| #(return_status, code) = module.set(locator, module_type, module_freq, module_size, module_form_factor) |
| if not return_status: |
| return (return_status, code) |
| memory_dict[node_id].append(module) |
| if code not in warning_text: |
| warning_text += code |
| node_id += 1 #TODO revise. Added for allowing VM as compute hosts |
| |
| #Fill memory nodes |
| #Hugepage size is constant for all nodes |
| hugepage_sz = get_hugepage_size(ssh_conn) |
| for node_id, modules in memory_dict.iteritems(): |
| memory_node = MemoryNode() |
| memory_node.set(modules, hugepage_sz, get_hugepage_nr(ssh_conn,hugepage_sz, node_id)) |
| memory_nodes[node_id] = memory_node |
| |
| return (True, warning_text) |
| |
| def get_cpu_topology_ht(ssh_conn, topology): |
| command = 'cat /proc/cpuinfo' |
| (_, stdout, stderr) = ssh_conn.exec_command(command) |
| error = stderr.read() |
| if len(error)>0: |
| raise paramiko.ssh_exception.SSHException(command +' : '+ error) |
| sockets = [] |
| cores = [] |
| core_map = {} |
| core_details = [] |
| core_lines = {} |
| for line in stdout.readlines(): |
| if len(line.strip()) != 0: |
| name, value = line.split(":", 1) |
| core_lines[name.strip()] = value.strip() |
| else: |
| core_details.append(core_lines) |
| core_lines = {} |
| |
| for core in core_details: |
| for field in ["processor", "core id", "physical id"]: |
| if field not in core: |
| return(False,'Error getting '+field+' value from /proc/cpuinfo') |
| core[field] = int(core[field]) |
| |
| if core["core id"] not in cores: |
| cores.append(core["core id"]) |
| if core["physical id"] not in sockets: |
| sockets.append(core["physical id"]) |
| key = (core["physical id"], core["core id"]) |
| if key not in core_map: |
| core_map[key] = [] |
| core_map[key].append(core["processor"]) |
| |
| for s in sockets: |
| hyperthreaded_cores = list() |
| for c in cores: |
| hyperthreaded_cores.append(core_map[(s,c)]) |
| topology[s] = hyperthreaded_cores |
| |
| return (True, "") |
| |
| def get_processor_information(ssh_conn, vish_conn, processors): |
| warning_text="" |
| #Processor features are the same for all processors |
| #TODO (at least using virsh capabilities)nr_numa_nodes |
| capabilities = list() |
| tree=ElementTree.fromstring(vish_conn.getCapabilities()) |
| for target in tree.findall("host/cpu/feature"): |
| if target.get("name") == 'pdpe1gb': |
| capabilities.append('lps') |
| elif target.get("name") == 'dca': |
| capabilities.append('dioc') |
| elif target.get("name") == 'vmx' or target.get("name") == 'svm': |
| capabilities.append('hwsv') |
| elif target.get("name") == 'ht': |
| capabilities.append('ht') |
| |
| target = tree.find("host/cpu/arch") |
| if target.text == 'x86_64' or target.text == 'amd64': |
| capabilities.append('64b') |
| |
| command = 'cat /proc/cpuinfo | grep flags' |
| (_, stdout, stderr) = ssh_conn.exec_command(command) |
| error = stderr.read() |
| if len(error)>0: |
| raise paramiko.ssh_exception.SSHException(command +' : '+ error) |
| line = stdout.readline() |
| if 'ept' in line or 'npt' in line: |
| capabilities.append('tlbps') |
| |
| #Find out if IOMMU is enabled |
| command = 'dmesg |grep -e Intel-IOMMU' |
| (_, stdout, stderr) = ssh_conn.exec_command(command) |
| error = stderr.read() |
| if len(error)>0: |
| raise paramiko.ssh_exception.SSHException(command +' : '+ error) |
| if 'enabled' in stdout.read(): |
| capabilities.append('iommu') |
| |
| #Equivalent for AMD |
| command = 'dmesg |grep -e AMD-Vi' |
| (_, stdout, stderr) = ssh_conn.exec_command(command) |
| error = stderr.read() |
| if len(error)>0: |
| raise paramiko.ssh_exception.SSHException(command +' : '+ error) |
| if len(stdout.read()) > 0: |
| capabilities.append('iommu') |
| |
| #----------------------------------------------------------- |
| topology = dict() |
| #In case hyperthreading is active it is necessary to determine cpu topology using /proc/cpuinfo |
| if 'ht' in capabilities: |
| (return_status, code) = get_cpu_topology_ht(ssh_conn, topology) |
| if not return_status: |
| return (return_status, code) |
| warning_text += code |
| |
| #Otherwise it is possible to do it using virsh capabilities |
| else: |
| for target in tree.findall("host/topology/cells/cell"): |
| socket_id = int(target.get("id")) |
| topology[socket_id] = list() |
| for cpu in target.findall("cpus/cpu"): |
| topology[socket_id].append(int(cpu.get("id"))) |
| |
| #----------------------------------------------------------- |
| #Create a dictionary with the information of all processors |
| #p_fam = p_man = p_ver = None |
| tree=ElementTree.fromstring(vish_conn.getSysinfo(0)) |
| #print vish_conn.getSysinfo(0) |
| #return (False, 'forces error for debuging') |
| not_populated=False |
| socket_id = -1 #in case we can not determine the socket_id we assume incremental order, starting by 0 |
| for target in tree.findall("processor"): |
| count = 0 |
| socket_id += 1 |
| #Get processor id, family, manufacturer and version |
| for entry in target.findall("entry"): |
| if entry.get("name") == "status": |
| if entry.text[0:11] == "Unpopulated": |
| not_populated=True |
| elif entry.get("name") == 'socket_destination': |
| socket_text = entry.text |
| if socket_text.startswith('CPU'): |
| socket_text = socket_text.strip('CPU') |
| socket_text = socket_text.strip() #removes trailing spaces |
| if socket_text.isdigit() and int(socket_text)<9 and int(socket_text)>0: |
| socket_id = int(socket_text) - 1 |
| |
| elif entry.get("name") == 'family': |
| family = entry.text |
| count += 1 |
| elif entry.get("name") == 'manufacturer': |
| manufacturer = entry.text |
| count += 1 |
| elif entry.get("name") == 'version': |
| version = entry.text.strip() |
| count += 1 |
| if count != 3: |
| return (False, 'Error. Not all expected fields could be found in processor') |
| |
| #Create and fill processor structure |
| if not_populated: |
| continue #avoid inconsistence of some machines where more socket detected than |
| processor = ProcessorNode() |
| (return_status, code) = processor.set(socket_id, family, manufacturer, version, capabilities, topology[socket_id]) |
| if not return_status: |
| return (return_status, code) |
| if code not in warning_text: |
| warning_text += code |
| |
| #Add processor to the processors dictionary |
| processors[socket_id] = processor |
| |
| return (True, warning_text) |
| |
| def get_nic_information(ssh_conn, virsh_conn, nic_topology): |
| warning_text="" |
| #Get list of net devices |
| net_devices = virsh_conn.listDevices('net',0) |
| print virsh_conn.listDevices('net',0) |
| for device in net_devices: |
| try: |
| #Get the XML descriptor of the device: |
| net_XML = ElementTree.fromstring(virsh_conn.nodeDeviceLookupByName(device).XMLDesc(0)) |
| #print "net_XML:" , net_XML |
| #obtain the parent |
| parent = net_XML.find('parent') |
| if parent == None: |
| print 'No parent was found in XML for device '+device |
| #Error. continue?------------------------------------------------------------- |
| continue |
| if parent.text == 'computer': |
| continue |
| if not parent.text.startswith('pci_'): |
| print device + ' parent is neither computer nor pci' |
| #Error. continue?------------------------------------------------------------- |
| continue |
| interface = net_XML.find('capability/interface').text |
| mac = net_XML.find('capability/address').text |
| |
| #Get the pci XML |
| pci_XML = ElementTree.fromstring(virsh_conn.nodeDeviceLookupByName(parent.text).XMLDesc(0)) |
| #print pci_XML |
| #Get pci |
| name = pci_XML.find('name').text.split('_') |
| pci = name[1]+':'+name[2]+':'+name[3]+'.'+name[4] |
| |
| #If slot == 0 it is a PF, otherwise it is a VF |
| capability = pci_XML.find('capability') |
| if capability.get('type') != 'pci': |
| print device + 'Capability is not of type pci in '+parent.text |
| #Error. continue?------------------------------------------------------------- |
| continue |
| slot = capability.find('slot').text |
| bus = capability.find('bus').text |
| node_id = None |
| numa_ = capability.find('numa') |
| if numa_ != None: |
| node_id = numa_.get('node'); |
| if node_id != None: node_id =int(node_id) |
| if slot == None or bus == None: |
| print device + 'Bus and slot not detected in '+parent.text |
| #Error. continue?------------------------------------------------------------- |
| continue |
| if slot != '0': |
| # print ElementTree.tostring(pci_XML) |
| virtual = True |
| capability_pf = capability.find('capability') |
| if capability_pf.get('type') != 'phys_function': |
| print 'physical_function not found in VF '+parent.text |
| #Error. continue?------------------------------------------------------------- |
| continue |
| PF_pci = capability_pf.find('address').attrib |
| PF_pci_text = PF_pci['domain'].split('x')[1]+':'+PF_pci['bus'].split('x')[1]+':'+PF_pci['slot'].split('x')[1]+'.'+PF_pci['function'].split('x')[1] |
| |
| else: |
| virtual = False |
| |
| #Obtain node for the port |
| if node_id == None: |
| node_id = int(bus)>>6 |
| #print "node_id:", node_id |
| |
| #Only for non virtual interfaces: Obtain speed and if link is detected (this must be done using ethtool) |
| if not virtual: |
| command = 'sudo ethtool '+interface+' | grep -e Speed -e "Link detected"' |
| (_, stdout, stderr) = ssh_conn.exec_command(command) |
| error = stderr.read() |
| if len(error) >0: |
| print 'Error running '+command+'\n'+error |
| #Error. continue?------------------------------------------------------------- |
| continue |
| for line in stdout.readlines(): |
| line = line.strip().rstrip('\n').split(': ') |
| if line[0] == 'Speed': |
| if line[1].endswith('Mb/s'): |
| speed = int(line[1].split('M')[0])*int(1e6) |
| elif line[1].endswith('Gb/s'): |
| speed = int(line[1].split('G')[0])*int(1e9) |
| elif line[1].endswith('Kb/s'): |
| speed = int(line[1].split('K')[0])*int(1e3) |
| else: |
| #the interface is listed but won't be used |
| speed = 0 |
| elif line[0] == 'Link detected': |
| if line[1] == 'yes': |
| enabled = True |
| else: |
| enabled = False |
| else: |
| print 'Unnexpected output of command '+command+':' |
| print line |
| #Error. continue?------------------------------------------------------------- |
| continue |
| |
| if not node_id in nic_topology: |
| nic_topology[node_id] = list() |
| #With this implementation we make the RAD with only one nic per node and this nic has all ports, TODO: change this by including parent information of PF |
| nic_topology[node_id].append(Nic()) |
| |
| #Load the appropriate nic |
| nic = nic_topology[node_id][0] |
| |
| #Create a new port and fill it |
| port = Port() |
| port.name = interface |
| port.virtual = virtual |
| port.speed = speed |
| if virtual: |
| port.available_bw = 0 |
| port.PF_pci_device_id = PF_pci_text |
| else: |
| port.available_bw = speed |
| if speed == 0: |
| port.enabled = False |
| else: |
| port.enabled = enabled |
| |
| port.eligible = virtual #Only virtual ports are eligible |
| port.mac = mac |
| port.pci_device_id = pci |
| port.pci_device_id_split = name[1:] |
| |
| #Save the port information |
| nic.add_port(port) |
| except Exception,e: |
| print 'Error: '+str(e) |
| |
| #set in vitual ports if they are enabled |
| for nic in nic_topology.itervalues(): |
| for port in nic[0].ports.itervalues(): |
| # print port.pci_device_id |
| if port.virtual: |
| enabled = nic[0].ports.get(port.PF_pci_device_id) |
| if enabled == None: |
| return(False, 'The PF '+port.PF_pci_device_id+' (VF '+port.pci_device_id+') is not present in ports dict') |
| #Only if the PF is enabled the VF can be enabled |
| if nic[0].ports[port.PF_pci_device_id].enabled: |
| port.enabled = True |
| else: |
| port.enabled = False |
| |
| return (True, warning_text) |
| |
| def get_nic_information_old(ssh_conn, nic_topology): |
| command = 'lstopo-no-graphics --of xml' |
| (_, stdout, stderr) = ssh_conn.exec_command(command) |
| error = stderr.read() |
| if len(error)>0: |
| raise paramiko.ssh_exception.SSHException(command +' : '+ error) |
| tree=ElementTree.fromstring(stdout.read()) |
| for target in tree.findall("object/object"): |
| #Find numa nodes |
| if target.get("type") != "NUMANode": |
| continue |
| node_id = int(target.get("os_index")) |
| nic_topology[node_id] = list() |
| |
| #find nics in numa node |
| for entry in target.findall("object/object"): |
| if entry.get("type") != 'Bridge': |
| continue |
| nic_name = entry.get("name") |
| model = None |
| nic = Nic() |
| |
| #find ports in nic |
| for pcidev in entry.findall("object"): |
| if pcidev.get("type") != 'PCIDev': |
| continue |
| enabled = speed = mac = pci_busid = None |
| port = Port() |
| model = pcidev.get("name") |
| virtual = False |
| if 'Virtual' in model: |
| virtual = True |
| pci_busid = pcidev.get("pci_busid") |
| for osdev in pcidev.findall("object"): |
| name = osdev.get("name") |
| for info in osdev.findall("info"): |
| if info.get("name") != 'Address': |
| continue |
| mac = info.get("value") |
| #get the port speed and status |
| command = 'sudo ethtool '+name |
| (_, stdout, stderr) = ssh_conn.exec_command(command) |
| error = stderr.read() |
| if len(error)>0: |
| return (False, 'Error obtaining '+name+' information: '+error) |
| ethtool = stdout.read() |
| if '10000baseT/Full' in ethtool: |
| speed = 10e9 |
| elif '1000baseT/Full' in ethtool: |
| speed = 1e9 |
| elif '100baseT/Full' in ethtool: |
| speed = 100e6 |
| elif '10baseT/Full' in ethtool: |
| speed = 10e6 |
| else: |
| return (False, 'Speed not detected in '+name) |
| |
| enabled = False |
| if 'Link detected: yes' in ethtool: |
| enabled = True |
| |
| if speed != None and mac != None and pci_busid != None: |
| mac = mac.split(':') |
| pci_busid_split = re.split(':|\.', pci_busid) |
| #Fill the port information |
| port.set(name, virtual, enabled, speed, mac, pci_busid, pci_busid_split) |
| nic.add_port(port) |
| |
| if len(nic.ports) > 0: |
| #Fill the nic model |
| if model != None: |
| nic.set_model(model) |
| else: |
| nic.set_model(nic_name) |
| |
| #Add it to the topology |
| nic_topology[node_id].append(nic) |
| |
| return (True, "") |
| |
| def get_os_information(ssh_conn, os): |
| warning_text="" |
| # command = 'lsb_release -a' |
| # (stdin, stdout, stderr) = ssh_conn.exec_command(command) |
| # cont = 0 |
| # for line in stdout.readlines(): |
| # line_split = re.split('\t| *', line.rstrip('\n')) |
| # if line_split[0] == 'Distributor' and line_split[1] == 'ID:': |
| # distributor = line_split[2] |
| # cont += 1 |
| # elif line_split[0] == 'Release:': |
| # release = line_split[1] |
| # cont += 1 |
| # elif line_split[0] == 'Codename:': |
| # codename = line_split[1] |
| # cont += 1 |
| # if cont != 3: |
| # return (False, 'It was not possible to obtain the OS id') |
| # id_ = distributor+'-'+release+'-'+codename |
| |
| |
| command = 'cat /etc/redhat-release' |
| (_, stdout, _) = ssh_conn.exec_command(command) |
| id_text= stdout.read() |
| if len(id_text)==0: |
| #try with Ubuntu |
| command = 'lsb_release -d -s' |
| (_, stdout, _) = ssh_conn.exec_command(command) |
| id_text= stdout.read() |
| if len(id_text)==0: |
| raise paramiko.ssh_exception.SSHException("Can not determinte release neither with 'lsb_release' nor with 'cat /etc/redhat-release'") |
| id_ = id_text.rstrip('\n') |
| |
| command = 'uname -o' |
| (_, stdout, stderr) = ssh_conn.exec_command(command) |
| error = stderr.read() |
| if len(error)>0: |
| raise paramiko.ssh_exception.SSHException(command +' : '+ error) |
| type_ = stdout.read().rstrip('\n') |
| |
| command = 'uname -i' |
| (_, stdout, stderr) = ssh_conn.exec_command(command) |
| error = stderr.read() |
| if len(error)>0: |
| raise paramiko.ssh_exception.SSHException(command +' : '+ error) |
| bit_architecture = stdout.read().rstrip('\n') |
| |
| (return_status, code) = os.set(id_, type_, bit_architecture) |
| if not return_status: |
| return (return_status, code) |
| warning_text += code |
| return (True, warning_text) |
| |
| def get_hypervisor_information(virsh_conn, hypervisor): |
| type_ = virsh_conn.getType().rstrip('\n') |
| version = virsh_conn.getVersion() |
| lib_version = virsh_conn.getLibVersion() |
| |
| domains = list() |
| tree=ElementTree.fromstring(virsh_conn.getCapabilities()) |
| for target in tree.findall("guest"): |
| os_type = target.find("os_type").text |
| #We only allow full virtualization |
| if os_type != 'hvm': |
| continue |
| wordsize = int(target.find('arch/wordsize').text) |
| if wordsize == 64: |
| for domain in target.findall("arch/domain"): |
| domains.append(domain.get("type")) |
| |
| (return_status, code) = hypervisor.set(type_, version, lib_version, domains) |
| if not return_status: |
| return (return_status, code) |
| return (True, code) |
| |
| class RADavailableResourcesClass(RADclass): |
| def __init__(self, resources): |
| """Copy resources from the RADclass (server resources not taking into account resources used by VMs""" |
| #New |
| self.reserved = dict() #Dictionary of reserved resources for a server. Key are VNFC names and values RADreservedResources |
| self.cores_consumption = None #Dictionary of cpu consumption. Key is the cpu and the value is |
| |
| self.machine = resources.machine |
| self.user = resources.user |
| self.password = resources.password |
| self.name = resources.name |
| self.nr_processors = resources.nr_processors |
| self.processor_family = resources.processor_family |
| self.processor_manufacturer = resources.processor_manufacturer |
| self.processor_version = resources.processor_version |
| self.processor_features = resources.processor_features |
| self.memory_type = resources.memory_type |
| self.memory_freq = resources.memory_freq |
| self.memory_nr_channels = resources.memory_nr_channels |
| self.memory_size = resources.memory_size |
| self.memory_hugepage_sz = resources.memory_hugepage_sz |
| self.hypervisor = Hypervisor() |
| self.hypervisor.assign(resources.hypervisor) |
| self.os = OpSys() |
| self.os.assign(resources.os) |
| self.nodes = dict() |
| for node_k, node_v in resources.nodes.iteritems(): |
| self.nodes[node_k] = Node() |
| self.nodes[node_k].assign(node_v) |
| return |
| |
| def _get_cores_consumption_warnings(self): |
| """Returns list of warning strings in case warnings are generated. |
| In case no warnings are generated the return value will be an empty list""" |
| warnings = list() |
| #Get the cores consumption |
| (return_status, code) = get_ssh_connection(self.machine, self.user, self.password) |
| if not return_status: |
| return (return_status, code) |
| ssh_conn = code |
| command = 'mpstat -P ALL 1 1 | grep Average | egrep -v CPU\|all' |
| (_, stdout, stderr) = ssh_conn.exec_command(command) |
| error = stderr.read() |
| if len(error) > 0: |
| return (False, error) |
| |
| self.cores_consumption = dict() |
| for line in stdout.readlines(): |
| cpu_usage_split = re.split('\t| *', line.rstrip('\n')) |
| usage = 100 *(1 - float(cpu_usage_split[10])) |
| if usage > 0: |
| self.cores_consumption[int(cpu_usage_split[1])] = usage |
| ssh_conn.close() |
| #Check if any core marked as available in the nodes has cpu_usage > 0 |
| for _, node_v in self.nodes.iteritems(): |
| cores = node_v.processor.eligible_cores |
| for cpu in cores: |
| if len(cpu) > 1: |
| for core in cpu: |
| if core in self.cores_consumption: |
| warnings.append('Warning: Core '+str(core)+' is supposed to be idle but it is consuming '+str(self.cores_consumption[core])+'%') |
| else: |
| if cpu in self.cores_consumption: |
| warnings.append('Warning: Core '+str(core)+' is supposed to be idle but it is consuming '+str(self.cores_consumption[cpu])+'%') |
| |
| return warnings |
| |
| def reserved_to_text(self): |
| text = str() |
| for VNFC_name, VNFC_reserved in self.reserved.iteritems(): |
| text += ' VNFC: '+str(VNFC_name)+'\n' |
| text += VNFC_reserved.to_text() |
| |
| return text |
| |
| def obtain_usage(self): |
| resp = dict() |
| #Iterate through nodes to get cores, eligible cores, memory and physical ports (save ports usage for next section) |
| nodes = dict() |
| ports_usage = dict() |
| hugepage_size = dict() |
| for node_k, node_v in self.nodes.iteritems(): |
| node = dict() |
| ports_usage[node_k] = dict() |
| eligible_cores = list() |
| for pair in node_v.processor.eligible_cores: |
| if isinstance(pair, list): |
| for element in pair: |
| eligible_cores.append(element) |
| else: |
| eligible_cores.append(pair) |
| node['cpus'] = {'cores':node_v.processor.cores,'eligible_cores':eligible_cores} |
| node['memory'] = {'size':str(node_v.memory.node_size/(1024*1024*1024))+'GB','eligible':str(node_v.memory.eligible_memory/(1024*1024*1024))+'GB'} |
| hugepage_size[node_k] = node_v.memory.hugepage_sz |
| |
| ports = dict() |
| for nic in node_v.nic_list: |
| for port in nic.ports.itervalues(): |
| if port.enabled and not port.virtual: |
| ports[port.name] = {'speed':str(port.speed/1000000000)+'G'} |
| # print '*************** ',port.name,'speed',port.speed |
| ports_usage[node_k][port.name] = 100 - int(100*float(port.available_bw)/float(port.speed)) |
| node['ports'] = ports |
| nodes[node_k] = node |
| resp['RAD'] = nodes |
| |
| #Iterate through reserved section to get used cores, used memory and port usage |
| cores = dict() |
| memory = dict() |
| #reserved_cores = list |
| for node_k in self.nodes.iterkeys(): |
| if not node_k in cores: |
| cores[node_k] = list() |
| memory[node_k] = 0 |
| for _, reserved in self.reserved.iteritems(): |
| if node_k in reserved.node_reserved_resources: |
| node_v = reserved.node_reserved_resources[node_k] |
| cores[node_k].extend(node_v.reserved_cores) |
| memory[node_k] += node_v.reserved_hugepage_nr * hugepage_size[node_k] |
| |
| occupation = dict() |
| for node_k in self.nodes.iterkeys(): |
| ports = dict() |
| for name, usage in ports_usage[node_k].iteritems(): |
| ports[name] = {'occupied':str(usage)+'%'} |
| # print '****************cores',cores |
| # print '****************memory',memory |
| occupation[node_k] = {'cores':cores[node_k],'memory':str(memory[node_k]/(1024*1024*1024))+'GB','ports':ports} |
| resp['occupation'] = occupation |
| |
| return resp |
| |
| class RADreservedResources(): |
| def __init__(self): |
| self.node_reserved_resources = dict() #dict. keys are the RAD nodes id, values are NodeReservedResources |
| self.mgmt_interface_pci = None #pci in the VNF for the management interface |
| self.image = None #Path in remote machine of the VNFC image |
| |
| def update(self,reserved): |
| self.image = reserved.image |
| self.mgmt_interface_pci = reserved.mgmt_interface_pci |
| for k,v in reserved.node_reserved_resources.iteritems(): |
| if k in self.node_reserved_resources.keys(): |
| return (False, 'Duplicated node entry '+str(k)+' in reserved resources') |
| self.node_reserved_resources[k]=v |
| |
| return (True, "") |
| |
| def to_text(self): |
| text = ' image: '+str(self.image)+'\n' |
| for node_id, node_reserved in self.node_reserved_resources.iteritems(): |
| text += ' Node ID: '+str(node_id)+'\n' |
| text += node_reserved.to_text() |
| return text |
| |
| class NodeReservedResources(): |
| def __init__(self): |
| # reserved_shared_cores = None #list. List of all cores that the VNFC needs in shared mode #TODO Not used |
| # reserved_memory = None #Integer. Amount of KiB needed by the VNFC #TODO. Not used since hugepages are used |
| self.reserved_cores = list() #list. List of all cores that the VNFC uses |
| self.reserved_hugepage_nr = 0 #Integer. Number of hugepages needed by the VNFC |
| self.reserved_ports = dict() #dict. The key is the physical port pci and the value the VNFC port description |
| self.vlan_tags = dict() |
| self.cpu_pinning = None |
| |
| def to_text(self): |
| text = ' cores: '+str(self.reserved_cores)+'\n' |
| text += ' cpu_pinning: '+str(self.cpu_pinning)+'\n' |
| text += ' hugepages_nr: '+str(self.reserved_hugepage_nr)+'\n' |
| for port_pci, port_description in self.reserved_ports.iteritems(): |
| text += ' port: '+str(port_pci)+'\n' |
| text += port_description.to_text() |
| return text |
| |
| # def update(self,reserved): |
| # self.reserved_cores = list(reserved.reserved_cores) |
| # self.reserved_hugepage_nr = reserved.reserved_hugepage_nr |
| # self.reserved_ports = dict(reserved.reserved_ports) |
| # self.cpu_pinning = list(reserved.cpu_pinning) |
| |
| |
| |