From: tierno Date: Thu, 8 Sep 2016 06:17:37 +0000 (+0200) Subject: v0.4.7 remove folder utils moving files to parent to avoid an empty __utils.py__... X-Git-Tag: v1.0.0~11 X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2Fopenvim.git;a=commitdiff_plain;h=f053737a7b5dbf4b2ccee3c483da8371367dbeb7 v0.4.7 remove folder utils moving files to parent to avoid an empty __utils.py__ file without license Signed-off-by: tierno --- diff --git a/RADclass.py b/RADclass.py new file mode 100644 index 0000000..bfbdbaf --- /dev/null +++ b/RADclass.py @@ -0,0 +1,1619 @@ +# -*- coding: utf-8 -*- +import code + +## +# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. +# This file is part of openmano +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact with: nfvlabs@tid.es +## + +''' +Implement the logic for obtaining compute nodes information +Resource Availability Descriptor +''' +__author__="Pablo Montes" + +#TODO: remove warnings, remove unused things + +from definitionsClass import definitionsClass +from auxiliary_functions import get_ssh_connection +import libvirt +from xml.etree import ElementTree +import paramiko +import re +import yaml + + +def getCredentials(creds, data): + """Used as a backup for libvirt.openAuth in order to provide password that came with data, + not used by the moment + """ + print "RADclass:getCredentials", creds, data + for cred in creds: + print cred[1] + ": ", + if cred[0] == libvirt.VIR_CRED_AUTHNAME: + cred[4] = data + elif cred[0] == libvirt.VIR_CRED_PASSPHRASE: + cred[4] = data + else: + return -1 + return 0 + +class RADclass(): + def __init__(self): + self.name = None + self.machine = None + self.user = None + self.password = None + self.nodes = dict() #Dictionary of nodes. Keys are the node id, values are Node() elements + self.nr_processors = None #Integer. Number of processors in the system + self.processor_family = None #If all nodes have the same value equal them, otherwise keep as None + self.processor_manufacturer = None #If all nodes have the same value equal them, otherwise keep as None + self.processor_version = None #If all nodes have the same value equal them, otherwise keep as None + self.processor_features = None #If all nodes have the same value equal them, otherwise keep as None + self.memory_type = None #If all nodes have the same value equal them, otherwise keep as None + self.memory_freq = None #If all nodes have the same value equal them, otherwise keep as None + self.memory_nr_channels = None #If all nodes have the same value equal them, otherwise keep as None + self.memory_size = None #Integer. Sum of the memory in all nodes + self.memory_hugepage_sz = None + self.hypervisor = Hypervisor() #Hypervisor information + self.os = OpSys() #Operating system information + self.ports_list = list() #List containing all network ports in the node. This is used to avoid having defined multiple times the same port in the system + + + def obtain_RAD(self, user, password, machine): + """This function obtains the RAD information from the remote server. + It uses both a ssh and a libvirt connection. + It is desirable in future versions get rid of the ssh connection, but currently + libvirt does not provide all the needed information. + Returns (True, Warning) in case of success and (False, ) in case of error""" + warning_text="" + try: + #Get virsh and ssh connection + (return_status, code) = get_ssh_connection(machine, user, password) + if not return_status: + print 'RADclass.obtain_RAD() error:', code + return (return_status, code) + ssh_conn = code + + self.connection_IP = machine + #print "libvirt open pre" + virsh_conn=libvirt.open("qemu+ssh://"+user+'@'+machine+"/system") + #virsh_conn=libvirt.openAuth("qemu+ssh://"+user+'@'+machine+"/system", + # [[libvirt.VIR_CRED_AUTHNAME, libvirt.VIR_CRED_PASSPHRASE, libvirt.VIR_CRED_USERNAME], getCredentials, password], + # 0) + #print "libvirt open after" + + # #Set connection infomation + # (return_status, code) = self.set_connection_info(machine, user, password) + # if not return_status: + # return (return_status, 'Error in '+machine+': '+code) + + #Set server name + machine_name = get_hostname(virsh_conn) + (return_status, code) = self.set_name(machine_name) + if not return_status: + return (return_status, 'Error at self.set_name in '+machine+': '+code) + warning_text += code + + #Get the server processors information + processors = dict() + (return_status, code) = get_processor_information(ssh_conn, virsh_conn, processors) + if not return_status: + return (return_status, 'Error at get_processor_information in '+machine+': '+code) + warning_text += code + + #Get the server memory information + memory_nodes = dict() + (return_status, code) = get_memory_information(ssh_conn, virsh_conn, memory_nodes) + if not return_status: + return (return_status, 'Error at get_memory_information in '+machine+': '+code) + warning_text += code + + #Get nics information + nic_topology = dict() + # (return_status, code) = get_nic_information_old(ssh_conn, nic_topology) + (return_status, code) = get_nic_information(ssh_conn, virsh_conn, nic_topology) + if not return_status: + return (return_status, 'Error at get_nic_information in '+machine+': '+code) + warning_text += code + + #Pack each processor, memory node and nics in a node element + #and add the node to the RAD element + for socket_id, processor in processors.iteritems(): + node = Node() + if not socket_id in nic_topology: + nic_topology[socket_id] = list() + + (return_status, code) = node.set(processor, memory_nodes[socket_id], nic_topology[socket_id]) + # else: + # (return_status, code) = node.set(processor, memory_nodes[socket_id]) + if not return_status: + return (return_status, 'Error at node.set in '+machine+': '+code) + warning_text += code + (return_status, code) = self.insert_node(node) + if not return_status: + return (return_status, 'Error at self.insert_node in '+machine+': '+code) + if code not in warning_text: + warning_text += code + + #Fill os data + os = OpSys() + (return_status, code) = get_os_information(ssh_conn, os) + if not return_status: + return (return_status, 'Error at get_os_information in '+machine+': '+code) + warning_text += code + (return_status, code) = self.set_os(os) + if not return_status: + return (return_status, 'Error at self.set_os in '+machine+': '+code) + warning_text += code + + #Fill hypervisor data + hypervisor = Hypervisor() + (return_status, code) = get_hypervisor_information(virsh_conn, hypervisor) + if not return_status: + return (return_status, 'Error at get_hypervisor_information in '+machine+': '+code) + warning_text += code + (return_status, code) = self.set_hypervisor(hypervisor) + if not return_status: + return (return_status, 'Error at self.set_hypervisor in '+machine+': '+code) + warning_text += code + ssh_conn.close() + + return (True, warning_text) + except libvirt.libvirtError, e: + text = e.get_error_message() + print 'RADclass.obtain_RAD() exception:', text + return (False, text) + except paramiko.ssh_exception.SSHException, e: + text = e.args[0] + print "obtain_RAD ssh Exception:", text + return False, text + + def set_name(self,name): + """Sets the machine name. + Returns (True,Warning) in case of success and ('False',) in case of error""" + if not isinstance(name,str): + return (False, 'The variable \'name\' must be text') + self.name = name + return (True, "") + + def set_connection_info(self, machine, user, password): + """Sets the connection information. + Returns (True,Warning) in case of success and ('False',) in case of error""" + if not isinstance(machine,str): + return (False, 'The variable \'machine\' must be text') + if not isinstance(user,str): + return (False, 'The variable \'user\' must be text') +# if not isinstance(password,str): +# return (False, 'The variable \'password\' must be text') + (self.machine, self.user, self.password) = (machine, user, password) + return (True, "") + + def insert_node(self,node): + """Inserts a new node and updates class variables. + Returns (True,Warning) in case of success and ('False',) in case of error""" + if not isinstance(node,Node): + return (False, 'The variable \'node\' must be a Node element') + + if node.id_ in self.nodes: + return (False, 'The node is already present in the nodes list.') + + #Check if network ports have not been inserted previously as part of another node + for port_key in node.ports_list: + if port_key in self.ports_list: + return (False, 'Network port '+port_key+' defined multiple times in the system') + self.ports_list.append(port_key) + + #Insert the new node + self.nodes[node.id_] = node + + #update variables + self.update_variables() + + return (True, "") + + def update_variables(self): + """Updates class variables. + Returns (True,Warning) in case of success and ('False',) in case of error""" + warning_text="" + #The number of processors and nodes is the same + self.nr_processors = len(self.nodes) + + #If all processors are the same get the values. Otherwise keep them as none + prev_processor_family = prev_processor_manufacturer = prev_processor_version = prev_processor_features = None + different_processor_family = different_processor_manufacturer = different_processor_version = different_processor_features = False + for node in self.nodes.itervalues(): + (self.processor_family, self.processor_manufacturer, self.processor_version, self.processor_features) = node.get_processor_info() + if prev_processor_family != None and self.processor_family != prev_processor_family: + different_processor_family = True + if prev_processor_manufacturer != None and self.processor_manufacturer != prev_processor_manufacturer: + different_processor_manufacturer = True + if prev_processor_version != None and self.processor_version != prev_processor_version: + different_processor_version = True + if prev_processor_features != None and self.processor_features != prev_processor_features: + different_processor_features = True + (prev_processor_family, prev_processor_manufacturer, prev_processor_version, prev_processor_features) = (self.processor_family, self.processor_manufacturer, self.processor_version, self.processor_features) + + if different_processor_family: + self.processor_family = None + if different_processor_features: + self.processor_features = None + if different_processor_manufacturer: + self.processor_manufacturer = None + if different_processor_version: + self.processor_version = None + + #If all memory nodes are the same get the values. Otherwise keep them as none + #Sum the total memory + self.memory_size = 0 + different_memory_freq = different_memory_nr_channels = different_memory_type = different_memory_hugepage_sz = False + prev_memory_freq = prev_memory_nr_channels = prev_memory_type = prev_memory_hugepage_sz = None + for node in self.nodes.itervalues(): + (self.memory_freq, self.memory_nr_channels, self.memory_type, memory_size, self.memory_hugepage_sz) = node.get_memory_info() + self.memory_size += memory_size + if prev_memory_freq != None and self.memory_freq != prev_memory_freq: + different_memory_freq = True + if prev_memory_nr_channels != None and self.memory_nr_channels != prev_memory_nr_channels: + different_memory_nr_channels = True + if prev_memory_type != None and self.memory_type != prev_memory_type: + different_memory_type = True + if prev_memory_hugepage_sz != None and self.memory_hugepage_sz != prev_memory_hugepage_sz: + different_memory_hugepage_sz = True + (prev_memory_freq, prev_memory_nr_channels, prev_memory_type, prev_memory_hugepage_sz) = (self.memory_freq, self.memory_nr_channels, self.memory_type, self.memory_hugepage_sz) + + if different_memory_freq: + self.memory_freq = None + if different_memory_nr_channels: + self.memory_nr_channels = None + if different_memory_type: + self.memory_type = None + if different_memory_hugepage_sz: + warning_text += 'Detected different hugepages size in different sockets\n' + + return (True, warning_text) + + def set_hypervisor(self,hypervisor): + """Sets the hypervisor. + Returns (True,Warning) in case of success and ('False',) in case of error""" + if not isinstance(hypervisor,Hypervisor): + return (False, 'The variable \'hypervisor\' must be of class Hypervisor') + + self.hypervisor.assign(hypervisor) + return (True, "") + + def set_os(self,os): + """Sets the operating system. + Returns (True,Warning) in case of success and ('False',) in case of error""" + if not isinstance(os,OpSys): + return (False, 'The variable \'os\' must be of class OpSys') + + self.os.assign(os) + return (True, "") + + def to_text(self): + text= 'name: '+str(self.name)+'\n' + text+= 'processor:\n' + text+= ' nr_processors: '+str(self.nr_processors)+'\n' + text+= ' family: '+str(self.processor_family)+'\n' + text+= ' manufacturer: '+str(self.processor_manufacturer)+'\n' + text+= ' version: '+str(self.processor_version)+'\n' + text+= ' features: '+str(self.processor_features)+'\n' + text+= 'memory:\n' + text+= ' type: '+str(self.memory_type)+'\n' + text+= ' freq: '+str(self.memory_freq)+'\n' + text+= ' nr_channels: '+str(self.memory_nr_channels)+'\n' + text+= ' size: '+str(self.memory_size)+'\n' + text+= 'hypervisor:\n' + text+= self.hypervisor.to_text() + text+= 'os:\n' + text+= self.os.to_text() + text+= 'resource topology:\n' + text+= ' nr_nodes: '+ str(len(self.nodes))+'\n' + text+= ' nodes:\n' + for node_k, node_v in self.nodes.iteritems(): + text+= ' node'+str(node_k)+':\n' + text+= node_v.to_text() + return text + + def to_yaml(self): + return yaml.load(self.to_text()) + +class Node(): + def __init__(self): + self.id_ = None #Integer. Node id. Unique in the system + self.processor = ProcessorNode() #Information about the processor in the node + self.memory = MemoryNode() #Information about the memory in the node + self.nic_list = list() #List of Nic() containing information about the nics associated to the node + self.ports_list = list() #List containing all network ports in the node. This is used to avoid having defined multiple times the same port in the system + + def get_processor_info(self): + """Gets the processor information. Returns (processor_family, processor_manufacturer, processor_version, processor_features)""" + return self.processor.get_info() + + def get_memory_info(self): + """Gets the memory information. Returns (memory_freq, memory_nr_channels, memory_type, memory_size)""" + return self.memory.get_info() + +# def set(self, *args): +# """Sets the node information. Returns (True,Warning) in case of success and ('False',) in case of error""" +# if len(args)==2: +# processor = args[0] +# memory = args[1] +# nics = False +# elif len(args)==3: +# processor = args[0] +# memory = args[1] +# nic_list = args[2] +# nics = True +# else: +# return (False, 'Wrong number of elements calling Node().set()') + + def set(self, processor, memory, nic_list): + (status, return_code) = self.processor.assign(processor) + if not status: + return (status, return_code) + + self.id_ = processor.id_ + + (status, return_code) = self.memory.assign(memory) + if not status: + return (status, return_code) + +# if nics: + for nic in nic_list: + if not isinstance(nic,Nic): + return (False, 'The nics must be of type Nic') + self.nic_list.append(nic) + for port_key in nic.ports.iterkeys(): + if port_key in self.ports_list: + return (False, 'Network port '+port_key+'defined multiple times in the same node') + self.ports_list.append(port_key) + + return (True,"") + + def assign(self, node): + """Sets the node information. + Returns (True,Warning) in case of success and ('False',) in case of error""" + warning_text="" + processor = node.processor + memory = node.memory + nic_list = node.nic_list + (status, return_code) = self.processor.assign(processor) + if not status: + return (status, return_code) + + self.id_ = processor.id_ + + (status, return_code) = self.memory.assign(memory) + if not status: + return (status, return_code) + warning_text += code + + for nic in nic_list: + if not isinstance(nic,Nic): + return (False, 'The nics must be of type Nic') + self.nic_list.append(nic) + for port_key in nic.ports.iterkeys(): + if port_key in self.ports_list: + return (False, 'Network port '+port_key+'defined multiple times in the same node') + self.ports_list.append(port_key) + + return (True,warning_text) + + def to_text(self): + text= ' id: '+str(self.id_)+'\n' + text+= ' cpu:\n' + text += self.processor.to_text() + text+= ' memory:\n' + text += self.memory.to_text() + if len(self.nic_list) > 0: + text+= ' nics:\n' + nic_index = 0 + for nic in self.nic_list: + text+= ' nic '+str(nic_index)+':\n' + text += nic.to_text() + nic_index += 1 + return text + +class ProcessorNode(): + #Definition of the possible values of processor variables + possible_features = definitionsClass.processor_possible_features + possible_manufacturers = definitionsClass.processor_possible_manufacturers + possible_families = definitionsClass.processor_possible_families + possible_versions = definitionsClass.processor_possible_versions + + def __init__(self): + self.id_ = None #Integer. Numeric identifier of the socket + self.family = None #Text. Family name of the processor + self.manufacturer = None #Text. Manufacturer of the processor + self.version = None #Text. Model version of the processor + self.features = list() #list. List of features offered by the processor + self.cores = list() #list. List of cores in the processor. In case of hyperthreading the coupled cores are expressed as [a,b] + self.eligible_cores = list()#list. List of cores that can be used + #self.decicated_cores + #self.shared_cores -> this should also contain information to know if cores are being used + + def assign(self, processor): + """Sets the processor information. + Returns (True,Warning) in case of success and ('False',) in case of error""" + if not isinstance(processor,ProcessorNode): + return (False, 'The variable \'processor\' must be of class ProcessorNode') + + self.id_ = processor.id_ + self.family = processor.family + self.manufacturer = processor.manufacturer + self.version = processor.version + self.features = processor.features + self.cores = processor.cores + self.eligible_cores = processor.eligible_cores + + return (True, "") + + def set(self, id_, family, manufacturer, version, features, cores): + """Sets the processor information. + Returns (True,Warning) in case of success and ('False',) in case of error""" + warning_text = "" + + if not isinstance(id_,int): + return (False, 'The processor id_ must be of type int') + if not isinstance(family,str): + return (False, 'The processor family must be of type str') + if not isinstance(manufacturer,str): + return (False, 'The processor manufacturer must be of type str') + if not isinstance(version,str): + return (False, 'The processor version must be of type str') + if not isinstance(features,list): + return (False, 'The processor features must be of type list') + if not isinstance(cores,list): + return (False, 'The processor cores must be of type list') + + (self.id_, self.family, self.manufacturer, self.version) = (id_, family, manufacturer, version) + + if not manufacturer in self.possible_manufacturers: + warning_text += "processor manufacturer '%s' not among: %s\n" %(manufacturer, str(self.possible_manufacturers)) + if not family in self.possible_families: + warning_text += "family '%s' not among: %s\n" % (family, str(self.possible_families)) +# if not version in self.possible_versions: +# warning_text += 'The version %s is not one of these: %s\n' % (version, str(self.possible_versions)) + + for feature in features: + if not feature in self.possible_features: + warning_text += "processor feature '%s' not among: %s\n" % (feature, str(self.possible_versions)) + self.features.append(feature) + + for iterator in sorted(cores): + if not isinstance(iterator,list) or not all(isinstance(x, int) for x in iterator): + return (False, 'The cores list must be in the form of [[a,b],[c,d],...] where a,b,c,d are of type int') + self.cores.append(iterator) + + self.set_eligible_cores() + + return (True,warning_text) + + def set_eligible_cores(self): + """Set the default eligible cores, this is all cores non used by the host operating system""" + not_first = False + for iterator in self.cores: + if not_first: + self.eligible_cores.append(iterator) + else: + not_first = True + return + + def get_info(self): + """Returns processor parameters (self.family, self.manufacturer, self.version, self.features)""" + return (self.family, self.manufacturer, self.version, self.features) + + def to_text(self): + text= ' id: '+str(self.id_)+'\n' + text+= ' family: '+self.family+'\n' + text+= ' manufacturer: '+self.manufacturer+'\n' + text+= ' version: '+self.version+'\n' + text+= ' features: '+str(self.features)+'\n' + text+= ' cores: '+str(self.cores)+'\n' + text+= ' eligible_cores: '+str(self.eligible_cores)+'\n' + return text + +class MemoryNode(): + def __init__(self): + self.modules = list() #List of MemoryModule(). List of all modules installed in the node + self.nr_channels = None #Integer. Number of modules installed in the node + self.node_size = None #Integer. Total size in KiB of memory installed in the node + self.eligible_memory = None #Integer. Size in KiB of eligible memory in the node + self.hugepage_sz = None #Integer. Size in KiB of hugepages + self.hugepage_nr = None #Integer. Number of hugepages allocated in the module + self.eligible_hugepage_nr = None #Integer. Number of eligible hugepages in the node + self.type_ = None #Text. Type of memory modules. If modules have a different value keep it as None + self.freq = None #Integer. Frequency of the modules in MHz. If modules have a different value keep it as None + self.module_size = None #Integer. Size of the modules in KiB. If modules have a different value keep it as None + self.form_factor = None #Text. Form factor of the modules. If modules have a different value keep it as None + + def assign(self, memory_node): + return self.set(memory_node.modules, memory_node.hugepage_sz, memory_node.hugepage_nr) + + def set(self, modules, hugepage_sz, hugepage_nr): + """Set the memory node information. hugepage_sz must be expressed in KiB. + Returns (True,Warning) in case of success and ('False',) in case of error""" + if not isinstance(modules, list): + return (False, 'The modules must be a list of elements of class MemoryModule') + if not isinstance(hugepage_sz,int): + return (False, 'The hugepage_sz variable must be an int expressing the size in KiB') + if not isinstance(hugepage_nr,int): + return (False, 'The hugepage_nr variable must be of type int') + + (self.hugepage_sz, self.hugepage_nr) = (hugepage_sz, hugepage_nr) + self.node_size = self.nr_channels = 0 + + different_type = different_freq = different_module_size = different_form_factor = False + prev_type = prev_freq = prev_module_size = prev_form_factor = None + for iterator in modules: + if not isinstance(iterator,MemoryModule): + return (False, 'The modules must be a list of elements of class MemoryModule') + self.modules.append(iterator) + (self.type_, self.freq, self.module_size, self.form_factor) = (iterator.type_, iterator.freq, iterator.size, iterator.form_factor) + self.node_size += self.module_size + self.nr_channels += 1 + if prev_type != None and prev_type != self.type_: + different_type = True + if prev_freq != None and prev_freq != self.freq: + different_freq = True + if prev_module_size != None and prev_module_size != self.module_size: + different_module_size = True + if prev_form_factor != None and prev_form_factor != self.form_factor: + different_form_factor = True + (prev_type, prev_freq, prev_module_size, prev_form_factor) = (self.type_, self.freq, self.module_size, self.form_factor) + + if different_type: + self.type_ = None + if different_freq: + self.freq = None + if different_module_size: + self.module_size = None + if different_form_factor: + self.form_factor = None + + (return_value, error_code) = self.set_eligible_memory() + if not return_value: + return (return_value, error_code) + + return (True, "") + + def set_eligible_memory(self): + """Sets the default eligible_memory and eligible_hugepage_nr. This is all memory but 2GiB and all hugepages""" + self.eligible_memory = self.node_size - 2*1024*1024 + if self.eligible_memory < 0: + return (False, "There is less than 2GiB of memory in the module") + + self.eligible_hugepage_nr = self.hugepage_nr + return (True,"") + + def get_info(self): + """Return memory information (self.freq, self.nr_channels, self.type_, self.node_size)""" + return (self.freq, self.nr_channels, self.type_, self.node_size, self.hugepage_sz) + + def to_text(self): + text= ' node_size: '+str(self.node_size)+'\n' + text+= ' nr_channels: '+str(self.nr_channels)+'\n' + text+= ' eligible_memory: '+str(self.eligible_memory)+'\n' + text+= ' hugepage_sz: '+str(self.hugepage_sz)+'\n' + text+= ' hugepage_nr: '+str(self.hugepage_nr)+'\n' + text+= ' eligible_hugepage_nr: '+str(self.eligible_hugepage_nr)+'\n' + text+= ' type: '+self.type_+'\n' + text+= ' freq: '+str(self.freq)+'\n' + text+= ' module_size: '+str(self.module_size)+'\n' + text+= ' form_factor: '+self.form_factor+'\n' + text+= ' modules details:\n' + for module in self.modules: + text += module.to_text() + return text + +class MemoryModule(): + #Definition of the possible values of module variables + possible_types = definitionsClass.memory_possible_types + possible_form_factors = definitionsClass.memory_possible_form_factors + + def __init__(self): + self.locator = None #Text. Name of the memory module + self.type_ = None #Text. Type of memory module + self.freq = None #Integer. Frequency of the module in MHz + self.size = None #Integer. Size of the module in KiB + self.form_factor = None #Text. Form factor of the module + + def set(self, locator, type_, freq, size, form_factor): + """Sets the memory module information. + Frequency must be expressed in MHz and size in KiB. + Returns (True,Warning) in case of success and ('False',) in case of error""" + warning_text="" + if not isinstance(locator, str): + return (False, "The type of the variable locator must be str") + if not isinstance(type_, str): + return (False, "The type of the variable type_ must be str") + if not isinstance(form_factor, str): + return (False, "The type of the variable form_factor must be str") + if not isinstance(freq, int): + return (False, "The type of the variable freq must be int") + if not isinstance(size, int): + return (False, "The type of the variable size must be int") + + if not form_factor in self.possible_form_factors: + warning_text += "memory form_factor '%s' not among: %s\n" %(form_factor, str(self.possible_form_factors)) + if not type_ in self.possible_types: + warning_text += "memory type '%s' not among: %s\n" %(type_, str(self.possible_types)) + + (self.locator, self.type_, self.freq, self.size, self.form_factor) = (locator, type_, freq, size, form_factor) + return (True, warning_text) + + def to_text(self): + text= ' '+self.locator+':\n' + text+= ' type: '+self.type_+'\n' + text+= ' freq: '+str(self.freq)+'\n' + text+= ' size: '+str(self.size)+'\n' + text+= ' form factor: '+self.form_factor+'\n' + return text + +class Nic(): + def __init__(self): + self.model = None #Text. Model of the nic + self.ports = dict() #Dictionary of ports. Keys are the port name, value are Port() elements + + def set_model(self, model): + """Sets the model of the nic. Returns (True,Warning) in case of success and ('False',) in case of error""" + if not isinstance(model,str): + return (False, 'The \'model\' must be of type str') + + self.model = model + return (True, "") + + def add_port(self, port): + """Adds a port to the nic. Returns (True,Warning) in case of success and ('False',) in case of error""" + if not isinstance(port,Port): + return (False, 'The \'port\' must be of class Port') + +# port_id = str(port.pci_device_id[0])+':'+str(port.pci_device_id[1])+':'+str(port.pci_device_id[2])+'.'+str(port.pci_device_id[3]) +#CHANGED +# port_id = port.name + port_id = port.pci_device_id +#CHANGED END + if port_id in self.ports: + return (False, 'The \'port\' '+port.pci_device_id+' is duplicated in the nic') +# return (False, 'The \'port\' is duplicated in the nic') + + self.ports[port_id] = port + return (True, "") + + def to_text(self): + text= ' model: '+ str(self.model)+'\n' + text+= ' ports: '+'\n' + for key,port in self.ports.iteritems(): + text+= ' "'+key+'":'+'\n' + text += port.to_text() + return text + +class Port(): + def __init__(self): + self.name = None #Text. Port name + self.virtual = None #Boolean. States if the port is a virtual function + self.enabled = None #Boolean. States if the port is enabled + self.eligible = None #Boolean. States if the port is eligible + self.speed = None #Integer. Indicates the speed in Mbps + self.available_bw = None #Integer. BW in Mbps that is available. + self.mac = None #list. Indicates the mac address of the port as a list in format ['XX','XX','XX','XX','XX','XX'] + self.pci_device_id_split = None #list. Indicates the pci address of the port as a list in format ['XXXX','XX','XX','X'] + self.pci_device_id = None + self.PF_pci_device_id = None + +# def set(self, name, virtual, enabled, speed, mac, pci_device_id, pci_device_id_split): +# """Sets the port information. The variable speed indicates the speed in Mbps. Returns (True,Warning) in case of success and ('False',) in case of error""" +# if not isinstance(name,str): +# return (False, 'The variable \'name\' must be of type str') +# if not isinstance(virtual,bool): +# return (False, 'The variable \'virtual\' must be of type bool') +# if not isinstance(enabled,bool): +# return (False, 'The variable \'enabled\' must be of type bool') +# if not isinstance(enabled,bool): +# return (speed, 'The variable \'speed\' must be of type int') +# if not isinstance(mac, list) and not isinstance(mac,NoneType): +# return (False, 'The variable \'enabled\' must be of type list indicating the mac address in format [\'XXXX\',\'XX\',\'XX\',\'X\'] or NoneType') +# if not isinstance(pci_device_id_split, list) or len(pci_device_id_split) != 4: +# return (False, 'The variable \'pci_device_id_split\' must be of type list, indicating the pci address in format [\'XX\',\'XX\',\'XX\',\'XX\',\'XX\',\'XX\']') +# +# expected_len = [4,2,2,1] +# index = 0 +# for iterator in pci_device_id_split: +# if not isinstance(iterator,str) or not iterator.isdigit() or len(iterator) != expected_len[index]: +# return (False, 'The variable \'pci_device_id_split\' must be of type list, indicating the pci address in format [\'XX\',\'XX\',\'XX\',\'XX\',\'XX\',\'XX\']') +# index += 1 +# +# if not isinstance(mac,NoneType): +# for iterator in mac: +# if not isinstance(iterator,str) or not iterator.isalnum() or len(iterator) != 2: +# return (False, 'The variable \'enabled\' must be of type list indicating the mac address in format [\'XXXX\',\'XX\',\'XX\',\'X\'] or NoneType') +# +# #By default only virtual ports are eligible +# # (self.name, self.virtual, self.enabled, self.eligible, self.available_bw, self.speed, self.mac, self.pci_device_id, self.pci_device_id_split) = (name, virtual, enabled, virtual, speed, speed, mac, pci_device_id, pci_device_id_split) +# (self.name, self.virtual, self.enabled, self.eligible, self.available_bw, self.mac, self.pci_device_id, self.pci_device_id_split) = (name, virtual, enabled, virtual, speed, mac, pci_device_id, pci_device_id_split) + + def to_text(self): + text= ' pci: "'+ str(self.pci_device_id)+'"\n' + text+= ' virtual: '+ str(self.virtual)+'\n' + if self.virtual: + text+= ' PF_pci_id: "'+self.PF_pci_device_id+'"\n' + text+= ' eligible: '+ str(self.eligible)+'\n' + text+= ' enabled: '+str(self.enabled)+'\n' + text+= ' speed: '+ str(self.speed)+'\n' + text+= ' available bw: '+ str(self.available_bw)+'\n' + text+= ' mac: '+ str(self.mac)+'\n' + text+= ' source_name: '+ str(self.name)+'\n' + return text + +class Hypervisor(): + #Definition of the possible values of hypervisor variables + possible_types = definitionsClass.hypervisor_possible_types + possible_domain_types = definitionsClass.hypervisor_possible_domain_types + + def __init__(self): + self.type_ = None #Text. Hypervisor type_ + self.version = None #int. Hypervisor version + self.lib_version = None #int. Libvirt version used to compile hypervisor + self.domains = list() #list. List of all the available domains + + def set(self, hypervisor, version, lib_version, domains): + warning_text="" + if not isinstance(hypervisor,str): + return (False, 'The variable type_ must be of type str') + if not isinstance(version,int): + return (False, 'The variable version must be of type int') + if not isinstance(lib_version,int): + return (False, 'The library version must be of type int') + if not isinstance(domains,list): + return (False, 'Domains must be a list of the possible domains as str') + + if not hypervisor in self.possible_types: + warning_text += "Hyperpivor '%s' not among: %s\n" % (hypervisor, str(self.possible_types)) + + valid_domain_found = False + for domain in domains: + if not isinstance(domain,str): + return (False, 'Domains must be a list of the possible domains as str') + if domain in self.possible_domain_types: + valid_domain_found = True + self.domains.append(domain) + + if not valid_domain_found: + warning_text += 'No valid domain found among: %s\n' % str(self.possible_domain_types) + + + (self.version, self.lib_version, self.type_) = (version, lib_version, hypervisor) + return (True, warning_text) + + def assign(self, hypervisor): + (self.version, self.lib_version, self.type_) = (hypervisor.version, hypervisor.lib_version, hypervisor.type_) + for domain in hypervisor.domains: + self.domains.append(domain) + return + + def to_text(self): + text= ' type: '+self.type_+'\n' + text+= ' version: '+str(self.version)+'\n' + text+= ' libvirt version: '+ str(self.lib_version)+'\n' + text+= ' domains: '+str(self.domains)+'\n' + return text + +class OpSys(): + #Definition of the possible values of os variables + possible_id = definitionsClass.os_possible_id + possible_types = definitionsClass.os_possible_types + possible_architectures = definitionsClass.os_possible_architectures + + def __init__(self): + self.id_ = None #Text. Identifier of the OS. Formed by --. In linux this can be obtained using lsb_release -a + self.type_ = None #Text. Type of operating system + self.bit_architecture = None #Integer. Architecture + + def set(self, id_, type_, bit_architecture): + warning_text="" + if not isinstance(type_,str): + return (False, 'The variable type_ must be of type str') + if not isinstance(id_,str): + return (False, 'The variable id_ must be of type str') + if not isinstance(bit_architecture,str): + return (False, 'The variable bit_architecture must be of type str') + + if not type_ in self.possible_types: + warning_text += "os type '%s' not among: %s\n" %(type_, str(self.possible_types)) + if not id_ in self.possible_id: + warning_text += "os release '%s' not among: %s\n" %(id_, str(self.possible_id)) + if not bit_architecture in self.possible_architectures: + warning_text += "os bit_architecture '%s' not among: %s\n" % (bit_architecture, str(self.possible_architectures)) + + (self.id_, self.type_, self.bit_architecture) = (id_, type_, bit_architecture) + return (True, warning_text) + + def assign(self,os): + (self.id_, self.type_, self.bit_architecture) = (os.id_, os.type_, os.bit_architecture) + return + + def to_text(self): + text= ' id: '+self.id_+'\n' + text+= ' type: '+self.type_+'\n' + text+= ' bit_architecture: '+self.bit_architecture+'\n' + return text + +def get_hostname(virsh_conn): + return virsh_conn.getHostname().rstrip('\n') + +def get_hugepage_size(ssh_conn): + command = 'sudo hugeadm --page-sizes' +# command = 'hugeadm --page-sizes-all' + (_, stdout, stderr) = ssh_conn.exec_command(command) + error = stderr.read() + if len(error)>0: + raise paramiko.ssh_exception.SSHException(command +' : '+ error) + mem=stdout.read() + if mem=="": + return 0 + return int(mem) + +def get_hugepage_nr(ssh_conn,hugepage_sz, node_id): + command = 'cat /sys/devices/system/node/node'+str(node_id)+'/hugepages/hugepages-'+str(hugepage_sz/1024)+'kB/nr_hugepages' + (_, stdout, _) = ssh_conn.exec_command(command) + #print command, + #text = stdout.read() + #print "'"+text+"'" + #return int(text) + + try: + value=int(stdout.read()) + except: + value=0 + return value + +def get_memory_information(ssh_conn, virsh_conn, memory_nodes): + warning_text="" + tree=ElementTree.fromstring(virsh_conn.getSysinfo(0)) + memory_dict = dict() + node_id = 0 #TODO revise. Added for allowing VM as compute hosts + for target in tree.findall("memory_device"): + locator_f = size_f = freq_f = type_f = formfactor_f = False + locator_f = True #TODO revise. Added for allowing VM as compute hosts + module_form_factor = "" + for entry in target.findall("entry"): + if entry.get("name") == 'size': + size_f = True + size_split = entry.text.split(' ') + if size_split[1] == 'MB': + module_size = int(size_split[0]) * 1024 * 1024 + elif size_split[1] == 'GB': + module_size = int(size_split[0]) * 1024 * 1024 * 1024 + elif size_split[1] == 'KB': + module_size = int(size_split[0]) * 1024 + else: + module_size = int(size_split[0]) + + elif entry.get("name") == 'speed': + freq_f = True + freq_split = entry.text.split(' ') + if freq_split[1] == 'MHz': + module_freq = int(freq_split[0]) * 1024 * 1024 + elif freq_split[1] == 'GHz': + module_freq = int(freq_split[0]) * 1024 * 1024 * 1024 + elif freq_split[1] == 'KHz': + module_freq = int(freq_split[0]) * 1024 + + elif entry.get("name") == 'type': + type_f = True + module_type = entry.text + + elif entry.get("name") == 'form_factor': + formfactor_f = True + module_form_factor = entry.text + #TODO revise. Commented for allowing VM as compute hosts + # elif entry.get("name") == 'locator' and not locator_f: + # # other case, it is obtained by bank_locator that we give priority to + # locator = entry.text + # pos = locator.find(module_form_factor) + # if module_form_factor == locator[0:len(module_form_factor) ]: + # pos = len(module_form_factor) +1 + # else: + # pos = 0 + # if locator[pos] in "ABCDEFGH": + # locator_f = True + # node_id = ord(locator[pos])-ord('A') + # #print entry.text, node_id + # + # elif entry.get("name") == 'bank_locator': + # locator = entry.text + # pos = locator.find("NODE ") + # if pos >= 0 and len(locator)>pos+5: + # if locator[pos+5] in ("01234567"): #len("NODE ") is 5 + # node_id = int(locator[pos+5]) + # locator_f = True + # + + #When all module fields have been found add a new module to the list + if locator_f and size_f and freq_f and type_f and formfactor_f: + #If the memory node has not yet been created create it + if node_id not in memory_dict: + memory_dict[node_id] = [] + + #Add a new module to the memory node + module = MemoryModule() + #TODO revise. Changed for allowing VM as compute hosts + (return_status, code) = module.set('NODE %d' % node_id, module_type, module_freq, module_size, module_form_factor) + #(return_status, code) = module.set(locator, module_type, module_freq, module_size, module_form_factor) + if not return_status: + return (return_status, code) + memory_dict[node_id].append(module) + if code not in warning_text: + warning_text += code + node_id += 1 #TODO revise. Added for allowing VM as compute hosts + + #Fill memory nodes + #Hugepage size is constant for all nodes + hugepage_sz = get_hugepage_size(ssh_conn) + for node_id, modules in memory_dict.iteritems(): + memory_node = MemoryNode() + memory_node.set(modules, hugepage_sz, get_hugepage_nr(ssh_conn,hugepage_sz, node_id)) + memory_nodes[node_id] = memory_node + + return (True, warning_text) + +def get_cpu_topology_ht(ssh_conn, topology): + command = 'cat /proc/cpuinfo' + (_, stdout, stderr) = ssh_conn.exec_command(command) + error = stderr.read() + if len(error)>0: + raise paramiko.ssh_exception.SSHException(command +' : '+ error) + sockets = [] + cores = [] + core_map = {} + core_details = [] + core_lines = {} + for line in stdout.readlines(): + if len(line.strip()) != 0: + name, value = line.split(":", 1) + core_lines[name.strip()] = value.strip() + else: + core_details.append(core_lines) + core_lines = {} + + for core in core_details: + for field in ["processor", "core id", "physical id"]: + if field not in core: + return(False,'Error getting '+field+' value from /proc/cpuinfo') + core[field] = int(core[field]) + + if core["core id"] not in cores: + cores.append(core["core id"]) + if core["physical id"] not in sockets: + sockets.append(core["physical id"]) + key = (core["physical id"], core["core id"]) + if key not in core_map: + core_map[key] = [] + core_map[key].append(core["processor"]) + + for s in sockets: + hyperthreaded_cores = list() + for c in cores: + hyperthreaded_cores.append(core_map[(s,c)]) + topology[s] = hyperthreaded_cores + + return (True, "") + +def get_processor_information(ssh_conn, vish_conn, processors): + warning_text="" + #Processor features are the same for all processors + #TODO (at least using virsh capabilities)nr_numa_nodes + capabilities = list() + tree=ElementTree.fromstring(vish_conn.getCapabilities()) + for target in tree.findall("host/cpu/feature"): + if target.get("name") == 'pdpe1gb': + capabilities.append('lps') + elif target.get("name") == 'dca': + capabilities.append('dioc') + elif target.get("name") == 'vmx' or target.get("name") == 'svm': + capabilities.append('hwsv') + elif target.get("name") == 'ht': + capabilities.append('ht') + + target = tree.find("host/cpu/arch") + if target.text == 'x86_64' or target.text == 'amd64': + capabilities.append('64b') + + command = 'cat /proc/cpuinfo | grep flags' + (_, stdout, stderr) = ssh_conn.exec_command(command) + error = stderr.read() + if len(error)>0: + raise paramiko.ssh_exception.SSHException(command +' : '+ error) + line = stdout.readline() + if 'ept' in line or 'npt' in line: + capabilities.append('tlbps') + + #Find out if IOMMU is enabled + command = 'dmesg |grep -e Intel-IOMMU' + (_, stdout, stderr) = ssh_conn.exec_command(command) + error = stderr.read() + if len(error)>0: + raise paramiko.ssh_exception.SSHException(command +' : '+ error) + if 'enabled' in stdout.read(): + capabilities.append('iommu') + + #Equivalent for AMD + command = 'dmesg |grep -e AMD-Vi' + (_, stdout, stderr) = ssh_conn.exec_command(command) + error = stderr.read() + if len(error)>0: + raise paramiko.ssh_exception.SSHException(command +' : '+ error) + if len(stdout.read()) > 0: + capabilities.append('iommu') + + #----------------------------------------------------------- + topology = dict() + #In case hyperthreading is active it is necessary to determine cpu topology using /proc/cpuinfo + if 'ht' in capabilities: + (return_status, code) = get_cpu_topology_ht(ssh_conn, topology) + if not return_status: + return (return_status, code) + warning_text += code + + #Otherwise it is possible to do it using virsh capabilities + else: + for target in tree.findall("host/topology/cells/cell"): + socket_id = int(target.get("id")) + topology[socket_id] = list() + for cpu in target.findall("cpus/cpu"): + topology[socket_id].append(int(cpu.get("id"))) + + #----------------------------------------------------------- + #Create a dictionary with the information of all processors + #p_fam = p_man = p_ver = None + tree=ElementTree.fromstring(vish_conn.getSysinfo(0)) + #print vish_conn.getSysinfo(0) + #return (False, 'forces error for debuging') + not_populated=False + socket_id = -1 #in case we can not determine the socket_id we assume incremental order, starting by 0 + for target in tree.findall("processor"): + count = 0 + socket_id += 1 + #Get processor id, family, manufacturer and version + for entry in target.findall("entry"): + if entry.get("name") == "status": + if entry.text[0:11] == "Unpopulated": + not_populated=True + elif entry.get("name") == 'socket_destination': + socket_text = entry.text + if socket_text.startswith('CPU'): + socket_text = socket_text.strip('CPU') + socket_text = socket_text.strip() #removes trailing spaces + if socket_text.isdigit() and int(socket_text)<9 and int(socket_text)>0: + socket_id = int(socket_text) - 1 + + elif entry.get("name") == 'family': + family = entry.text + count += 1 + elif entry.get("name") == 'manufacturer': + manufacturer = entry.text + count += 1 + elif entry.get("name") == 'version': + version = entry.text.strip() + count += 1 + if count != 3: + return (False, 'Error. Not all expected fields could be found in processor') + + #Create and fill processor structure + if not_populated: + continue #avoid inconsistence of some machines where more socket detected than + processor = ProcessorNode() + (return_status, code) = processor.set(socket_id, family, manufacturer, version, capabilities, topology[socket_id]) + if not return_status: + return (return_status, code) + if code not in warning_text: + warning_text += code + + #Add processor to the processors dictionary + processors[socket_id] = processor + + return (True, warning_text) + +def get_nic_information(ssh_conn, virsh_conn, nic_topology): + warning_text="" + #Get list of net devices + net_devices = virsh_conn.listDevices('net',0) + print virsh_conn.listDevices('net',0) + for device in net_devices: + try: + #Get the XML descriptor of the device: + net_XML = ElementTree.fromstring(virsh_conn.nodeDeviceLookupByName(device).XMLDesc(0)) + #print "net_XML:" , net_XML + #obtain the parent + parent = net_XML.find('parent') + if parent == None: + print 'No parent was found in XML for device '+device + #Error. continue?------------------------------------------------------------- + continue + if parent.text == 'computer': + continue + if not parent.text.startswith('pci_'): + print device + ' parent is neither computer nor pci' + #Error. continue?------------------------------------------------------------- + continue + interface = net_XML.find('capability/interface').text + mac = net_XML.find('capability/address').text + + #Get the pci XML + pci_XML = ElementTree.fromstring(virsh_conn.nodeDeviceLookupByName(parent.text).XMLDesc(0)) + #print pci_XML + #Get pci + name = pci_XML.find('name').text.split('_') + pci = name[1]+':'+name[2]+':'+name[3]+'.'+name[4] + + #If slot == 0 it is a PF, otherwise it is a VF + capability = pci_XML.find('capability') + if capability.get('type') != 'pci': + print device + 'Capability is not of type pci in '+parent.text + #Error. continue?------------------------------------------------------------- + continue + slot = capability.find('slot').text + bus = capability.find('bus').text + node_id = None + numa_ = capability.find('numa') + if numa_ != None: + node_id = numa_.get('node'); + if node_id != None: node_id =int(node_id) + if slot == None or bus == None: + print device + 'Bus and slot not detected in '+parent.text + #Error. continue?------------------------------------------------------------- + continue + if slot != '0': + # print ElementTree.tostring(pci_XML) + virtual = True + capability_pf = capability.find('capability') + if capability_pf.get('type') != 'phys_function': + print 'physical_function not found in VF '+parent.text + #Error. continue?------------------------------------------------------------- + continue + PF_pci = capability_pf.find('address').attrib + PF_pci_text = PF_pci['domain'].split('x')[1]+':'+PF_pci['bus'].split('x')[1]+':'+PF_pci['slot'].split('x')[1]+'.'+PF_pci['function'].split('x')[1] + + else: + virtual = False + + #Obtain node for the port + if node_id == None: + node_id = int(bus)>>6 + #print "node_id:", node_id + + #Only for non virtual interfaces: Obtain speed and if link is detected (this must be done using ethtool) + if not virtual: + command = 'sudo ethtool '+interface+' | grep -e Speed -e "Link detected"' + (_, stdout, stderr) = ssh_conn.exec_command(command) + error = stderr.read() + if len(error) >0: + print 'Error running '+command+'\n'+error + #Error. continue?------------------------------------------------------------- + continue + for line in stdout.readlines(): + line = line.strip().rstrip('\n').split(': ') + if line[0] == 'Speed': + if line[1].endswith('Mb/s'): + speed = int(line[1].split('M')[0])*int(1e6) + elif line[1].endswith('Gb/s'): + speed = int(line[1].split('G')[0])*int(1e9) + elif line[1].endswith('Kb/s'): + speed = int(line[1].split('K')[0])*int(1e3) + else: + #the interface is listed but won't be used + speed = 0 + elif line[0] == 'Link detected': + if line[1] == 'yes': + enabled = True + else: + enabled = False + else: + print 'Unnexpected output of command '+command+':' + print line + #Error. continue?------------------------------------------------------------- + continue + + if not node_id in nic_topology: + nic_topology[node_id] = list() + #With this implementation we make the RAD with only one nic per node and this nic has all ports, TODO: change this by including parent information of PF + nic_topology[node_id].append(Nic()) + + #Load the appropriate nic + nic = nic_topology[node_id][0] + + #Create a new port and fill it + port = Port() + port.name = interface + port.virtual = virtual + port.speed = speed + if virtual: + port.available_bw = 0 + port.PF_pci_device_id = PF_pci_text + else: + port.available_bw = speed + if speed == 0: + port.enabled = False + else: + port.enabled = enabled + + port.eligible = virtual #Only virtual ports are eligible + port.mac = mac + port.pci_device_id = pci + port.pci_device_id_split = name[1:] + + #Save the port information + nic.add_port(port) + except Exception,e: + print 'Error: '+str(e) + + #set in vitual ports if they are enabled + for nic in nic_topology.itervalues(): + for port in nic[0].ports.itervalues(): +# print port.pci_device_id + if port.virtual: + enabled = nic[0].ports.get(port.PF_pci_device_id) + if enabled == None: + return(False, 'The PF '+port.PF_pci_device_id+' (VF '+port.pci_device_id+') is not present in ports dict') + #Only if the PF is enabled the VF can be enabled + if nic[0].ports[port.PF_pci_device_id].enabled: + port.enabled = True + else: + port.enabled = False + + return (True, warning_text) + +def get_nic_information_old(ssh_conn, nic_topology): + command = 'lstopo-no-graphics --of xml' + (_, stdout, stderr) = ssh_conn.exec_command(command) + error = stderr.read() + if len(error)>0: + raise paramiko.ssh_exception.SSHException(command +' : '+ error) + tree=ElementTree.fromstring(stdout.read()) + for target in tree.findall("object/object"): + #Find numa nodes + if target.get("type") != "NUMANode": + continue + node_id = int(target.get("os_index")) + nic_topology[node_id] = list() + + #find nics in numa node + for entry in target.findall("object/object"): + if entry.get("type") != 'Bridge': + continue + nic_name = entry.get("name") + model = None + nic = Nic() + + #find ports in nic + for pcidev in entry.findall("object"): + if pcidev.get("type") != 'PCIDev': + continue + enabled = speed = mac = pci_busid = None + port = Port() + model = pcidev.get("name") + virtual = False + if 'Virtual' in model: + virtual = True + pci_busid = pcidev.get("pci_busid") + for osdev in pcidev.findall("object"): + name = osdev.get("name") + for info in osdev.findall("info"): + if info.get("name") != 'Address': + continue + mac = info.get("value") + #get the port speed and status + command = 'sudo ethtool '+name + (_, stdout, stderr) = ssh_conn.exec_command(command) + error = stderr.read() + if len(error)>0: + return (False, 'Error obtaining '+name+' information: '+error) + ethtool = stdout.read() + if '10000baseT/Full' in ethtool: + speed = 10e9 + elif '1000baseT/Full' in ethtool: + speed = 1e9 + elif '100baseT/Full' in ethtool: + speed = 100e6 + elif '10baseT/Full' in ethtool: + speed = 10e6 + else: + return (False, 'Speed not detected in '+name) + + enabled = False + if 'Link detected: yes' in ethtool: + enabled = True + + if speed != None and mac != None and pci_busid != None: + mac = mac.split(':') + pci_busid_split = re.split(':|\.', pci_busid) + #Fill the port information + port.set(name, virtual, enabled, speed, mac, pci_busid, pci_busid_split) + nic.add_port(port) + + if len(nic.ports) > 0: + #Fill the nic model + if model != None: + nic.set_model(model) + else: + nic.set_model(nic_name) + + #Add it to the topology + nic_topology[node_id].append(nic) + + return (True, "") + +def get_os_information(ssh_conn, os): + warning_text="" +# command = 'lsb_release -a' +# (stdin, stdout, stderr) = ssh_conn.exec_command(command) +# cont = 0 +# for line in stdout.readlines(): +# line_split = re.split('\t| *', line.rstrip('\n')) +# if line_split[0] == 'Distributor' and line_split[1] == 'ID:': +# distributor = line_split[2] +# cont += 1 +# elif line_split[0] == 'Release:': +# release = line_split[1] +# cont += 1 +# elif line_split[0] == 'Codename:': +# codename = line_split[1] +# cont += 1 +# if cont != 3: +# return (False, 'It was not possible to obtain the OS id') +# id_ = distributor+'-'+release+'-'+codename + + + command = 'cat /etc/redhat-release' + (_, stdout, _) = ssh_conn.exec_command(command) + id_text= stdout.read() + if len(id_text)==0: + #try with Ubuntu + command = 'lsb_release -d -s' + (_, stdout, _) = ssh_conn.exec_command(command) + id_text= stdout.read() + if len(id_text)==0: + raise paramiko.ssh_exception.SSHException("Can not determinte release neither with 'lsb_release' nor with 'cat /etc/redhat-release'") + id_ = id_text.rstrip('\n') + + command = 'uname -o' + (_, stdout, stderr) = ssh_conn.exec_command(command) + error = stderr.read() + if len(error)>0: + raise paramiko.ssh_exception.SSHException(command +' : '+ error) + type_ = stdout.read().rstrip('\n') + + command = 'uname -i' + (_, stdout, stderr) = ssh_conn.exec_command(command) + error = stderr.read() + if len(error)>0: + raise paramiko.ssh_exception.SSHException(command +' : '+ error) + bit_architecture = stdout.read().rstrip('\n') + + (return_status, code) = os.set(id_, type_, bit_architecture) + if not return_status: + return (return_status, code) + warning_text += code + return (True, warning_text) + +def get_hypervisor_information(virsh_conn, hypervisor): + type_ = virsh_conn.getType().rstrip('\n') + version = virsh_conn.getVersion() + lib_version = virsh_conn.getLibVersion() + + domains = list() + tree=ElementTree.fromstring(virsh_conn.getCapabilities()) + for target in tree.findall("guest"): + os_type = target.find("os_type").text + #We only allow full virtualization + if os_type != 'hvm': + continue + wordsize = int(target.find('arch/wordsize').text) + if wordsize == 64: + for domain in target.findall("arch/domain"): + domains.append(domain.get("type")) + + (return_status, code) = hypervisor.set(type_, version, lib_version, domains) + if not return_status: + return (return_status, code) + return (True, code) + +class RADavailableResourcesClass(RADclass): + def __init__(self, resources): + """Copy resources from the RADclass (server resources not taking into account resources used by VMs""" + #New + self.reserved = dict() #Dictionary of reserved resources for a server. Key are VNFC names and values RADreservedResources + self.cores_consumption = None #Dictionary of cpu consumption. Key is the cpu and the value is + + self.machine = resources.machine + self.user = resources.user + self.password = resources.password + self.name = resources.name + self.nr_processors = resources.nr_processors + self.processor_family = resources.processor_family + self.processor_manufacturer = resources.processor_manufacturer + self.processor_version = resources.processor_version + self.processor_features = resources.processor_features + self.memory_type = resources.memory_type + self.memory_freq = resources.memory_freq + self.memory_nr_channels = resources.memory_nr_channels + self.memory_size = resources.memory_size + self.memory_hugepage_sz = resources.memory_hugepage_sz + self.hypervisor = Hypervisor() + self.hypervisor.assign(resources.hypervisor) + self.os = OpSys() + self.os.assign(resources.os) + self.nodes = dict() + for node_k, node_v in resources.nodes.iteritems(): + self.nodes[node_k] = Node() + self.nodes[node_k].assign(node_v) + return + + def _get_cores_consumption_warnings(self): + """Returns list of warning strings in case warnings are generated. + In case no warnings are generated the return value will be an empty list""" + warnings = list() + #Get the cores consumption + (return_status, code) = get_ssh_connection(self.machine, self.user, self.password) + if not return_status: + return (return_status, code) + ssh_conn = code + command = 'mpstat -P ALL 1 1 | grep Average | egrep -v CPU\|all' + (_, stdout, stderr) = ssh_conn.exec_command(command) + error = stderr.read() + if len(error) > 0: + return (False, error) + + self.cores_consumption = dict() + for line in stdout.readlines(): + cpu_usage_split = re.split('\t| *', line.rstrip('\n')) + usage = 100 *(1 - float(cpu_usage_split[10])) + if usage > 0: + self.cores_consumption[int(cpu_usage_split[1])] = usage + ssh_conn.close() + #Check if any core marked as available in the nodes has cpu_usage > 0 + for _, node_v in self.nodes.iteritems(): + cores = node_v.processor.eligible_cores + for cpu in cores: + if len(cpu) > 1: + for core in cpu: + if core in self.cores_consumption: + warnings.append('Warning: Core '+str(core)+' is supposed to be idle but it is consuming '+str(self.cores_consumption[core])+'%') + else: + if cpu in self.cores_consumption: + warnings.append('Warning: Core '+str(core)+' is supposed to be idle but it is consuming '+str(self.cores_consumption[cpu])+'%') + + return warnings + + def reserved_to_text(self): + text = str() + for VNFC_name, VNFC_reserved in self.reserved.iteritems(): + text += ' VNFC: '+str(VNFC_name)+'\n' + text += VNFC_reserved.to_text() + + return text + + def obtain_usage(self): + resp = dict() + #Iterate through nodes to get cores, eligible cores, memory and physical ports (save ports usage for next section) + nodes = dict() + ports_usage = dict() + hugepage_size = dict() + for node_k, node_v in self.nodes.iteritems(): + node = dict() + ports_usage[node_k] = dict() + eligible_cores = list() + for pair in node_v.processor.eligible_cores: + if isinstance(pair, list): + for element in pair: + eligible_cores.append(element) + else: + eligible_cores.append(pair) + node['cpus'] = {'cores':node_v.processor.cores,'eligible_cores':eligible_cores} + node['memory'] = {'size':str(node_v.memory.node_size/(1024*1024*1024))+'GB','eligible':str(node_v.memory.eligible_memory/(1024*1024*1024))+'GB'} + hugepage_size[node_k] = node_v.memory.hugepage_sz + + ports = dict() + for nic in node_v.nic_list: + for port in nic.ports.itervalues(): + if port.enabled and not port.virtual: + ports[port.name] = {'speed':str(port.speed/1000000000)+'G'} +# print '*************** ',port.name,'speed',port.speed + ports_usage[node_k][port.name] = 100 - int(100*float(port.available_bw)/float(port.speed)) + node['ports'] = ports + nodes[node_k] = node + resp['RAD'] = nodes + + #Iterate through reserved section to get used cores, used memory and port usage + cores = dict() + memory = dict() + #reserved_cores = list + for node_k in self.nodes.iterkeys(): + if not node_k in cores: + cores[node_k] = list() + memory[node_k] = 0 + for _, reserved in self.reserved.iteritems(): + if node_k in reserved.node_reserved_resources: + node_v = reserved.node_reserved_resources[node_k] + cores[node_k].extend(node_v.reserved_cores) + memory[node_k] += node_v.reserved_hugepage_nr * hugepage_size[node_k] + + occupation = dict() + for node_k in self.nodes.iterkeys(): + ports = dict() + for name, usage in ports_usage[node_k].iteritems(): + ports[name] = {'occupied':str(usage)+'%'} +# print '****************cores',cores +# print '****************memory',memory + occupation[node_k] = {'cores':cores[node_k],'memory':str(memory[node_k]/(1024*1024*1024))+'GB','ports':ports} + resp['occupation'] = occupation + + return resp + +class RADreservedResources(): + def __init__(self): + self.node_reserved_resources = dict() #dict. keys are the RAD nodes id, values are NodeReservedResources + self.mgmt_interface_pci = None #pci in the VNF for the management interface + self.image = None #Path in remote machine of the VNFC image + + def update(self,reserved): + self.image = reserved.image + self.mgmt_interface_pci = reserved.mgmt_interface_pci + for k,v in reserved.node_reserved_resources.iteritems(): + if k in self.node_reserved_resources.keys(): + return (False, 'Duplicated node entry '+str(k)+' in reserved resources') + self.node_reserved_resources[k]=v + + return (True, "") + + def to_text(self): + text = ' image: '+str(self.image)+'\n' + for node_id, node_reserved in self.node_reserved_resources.iteritems(): + text += ' Node ID: '+str(node_id)+'\n' + text += node_reserved.to_text() + return text + +class NodeReservedResources(): + def __init__(self): + # reserved_shared_cores = None #list. List of all cores that the VNFC needs in shared mode #TODO Not used + # reserved_memory = None #Integer. Amount of KiB needed by the VNFC #TODO. Not used since hugepages are used + self.reserved_cores = list() #list. List of all cores that the VNFC uses + self.reserved_hugepage_nr = 0 #Integer. Number of hugepages needed by the VNFC + self.reserved_ports = dict() #dict. The key is the physical port pci and the value the VNFC port description + self.vlan_tags = dict() + self.cpu_pinning = None + + def to_text(self): + text = ' cores: '+str(self.reserved_cores)+'\n' + text += ' cpu_pinning: '+str(self.cpu_pinning)+'\n' + text += ' hugepages_nr: '+str(self.reserved_hugepage_nr)+'\n' + for port_pci, port_description in self.reserved_ports.iteritems(): + text += ' port: '+str(port_pci)+'\n' + text += port_description.to_text() + return text + +# def update(self,reserved): +# self.reserved_cores = list(reserved.reserved_cores) +# self.reserved_hugepage_nr = reserved.reserved_hugepage_nr +# self.reserved_ports = dict(reserved.reserved_ports) +# self.cpu_pinning = list(reserved.cpu_pinning) + + + diff --git a/auxiliary_functions.py b/auxiliary_functions.py new file mode 100644 index 0000000..44e7fcb --- /dev/null +++ b/auxiliary_functions.py @@ -0,0 +1,227 @@ +# -*- coding: utf-8 -*- + +## +# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. +# This file is part of openmano +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact with: nfvlabs@tid.es +## + +''' +Common usuful functions +''' + +__author__="Alfonso Tierno, Pablo Montes" +__date__ ="$10-jul-2014 12:07:15$" + + +import yaml +import paramiko +from definitionsClass import definitionsClass +from definitionsClass import Units +import random +from jsonschema import validate as js_v, exceptions as js_e + +def check_and_convert_units(value, value_type): + """TODO: Update description + This method receives a text with 2 fields using a blank as separator and a list of valid units. The first field must represent a number + and the second one units. + In case the second field is not one of valid_units (False, ) is returned. + In case the second field is a valid unit the first number is converted in the following way: + Gbps, Mbps, kbps -> Mbps + GB,MB,KB,B,GiB,MiB,KiB -> B + GHz,MHz,KHz,Hz -> Hz + If conversion is done successfully (True, ) is returned""" + try: + if value_type == Units.no_units: + if not isinstance(value,int) and not isinstance(value,float): + return (False, 'When no units are used only an integer or float must be used') + elif value_type == Units.name: + if not isinstance(value,str): + return (False, 'For names str must be used') + elif value_type == Units.boolean: + if not isinstance(value,bool): + return (False, 'A boolean or Yes/No mut be used') + else: + splitted = value.split(' ') + if len(splitted) != 2: + return (False, 'Expected format: ') + (value, units) = splitted + if ',' in value or '.' in value: + return (False, 'Use integers to represent numeric values') + + value = int(value) + +# if not isinstance(value_type, Units): +# return (False, 'Not valid value_type') + + valid_units = definitionsClass.units[value_type] + + #Convert everything to upper in order to make comparations easier + units = units.upper() + for i in range(0, len(valid_units)): + valid_units[i] = valid_units[i].upper() + + #Check the used units are valid ones + if units not in valid_units: + return (False, 'Valid units are: '+', '.join(valid_units)) + + if units.startswith('GI'): + value = value *1024*1024*1024 + elif units.startswith('MI'): + value = value *1024*1024 + elif units.startswith('KI'): + value = value *1024 + elif units.startswith('G'): + value = value *1000000000 + elif units.startswith('M'): + value = value *1000000 + elif units.startswith('K'): + value = value *1000 + except Exception,e: + return (False, 'Unexpected error in auxiliary_functions.py - check_and_convert_units:\n'+str(e)) + + return (True, value) + +def get_ssh_connection(machine, user=None, password=None): + """Stablishes an ssh connection to the remote server. Returns (True, paramiko_ssh) in case of success or (False, ) in case of error""" + try: + s = paramiko.SSHClient() + s.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + s.load_system_host_keys() + s.connect(machine, 22, user, password, timeout=10) + except Exception,e: + return (False, 'It was not possible to connect to '+machine+str(e)) + + return (True, s) + +def run_in_remote_server(s,command): + """Runs in the remote server the specified command. Returns (True, stdout) in case of success or (False, ) in case of error""" + try: + (_, stdout, stderr) = s.exec_command(command) + error_msg = stderr.read() + if len(error_msg) > 0: + return (False, error_msg) + except Exception,e: + return (False, str(e)) + + return (True, stdout) + +def read_file(file_): + """Reads a file specified by 'file' and returns (True,) in case of success or (False, ) in case of failure""" + try: + f = open(file_, 'r') + read_data = f.read() + f.close() + except Exception,e: + return (False, str(e)) + + return (True, read_data) + +def check_contains(element, keywords): + """Auxiliary function used to check if a yaml structure contains or not + an specific field. Returns a bool""" + for key in keywords: + if not key in element: + return False + return True + +def check_contains_(element, keywords): + """Auxiliary function used to check if a yaml structure contains or not + an specific field. Returns a bool,missing_variables""" + for key in keywords: + if not key in element: + return False, key + return True, None + +def write_file(file_, content): + """Generates a file specified by 'file' and fills it using 'content'""" + f = open(file_, 'w') + f.write(content) + f.close() + +def nice_print(yaml_element): + """Print a yaml structure. Used mainly for debugging""" + print(yaml.dump(yaml_element, default_flow_style=False)) + +def new_random_mac(): + mac = (0xE2, random.randint(0x00, 0xff), random.randint(0x00, 0xff), random.randint(0x00, 0xff), random.randint(0x00, 0xff), random.randint(0x00, 0xff) ) + return ':'.join(map(lambda x: "%02X" % x, mac)) + +def parse_dict(var, template): + if type(var) is not dict: return -1, 'not a dictionary' + for _,tv in template.items(): + if type(tv) is list: + return + +def delete_nulls(var): + if type(var) is dict: + for k in var.keys(): + if var[k] is None: del var[k] + elif type(var[k]) is dict or type(var[k]) is list or type(var[k]) is tuple: + if delete_nulls(var[k]): del var[k] + if len(var) == 0: return True + elif type(var) is list or type(var) is tuple: + for k in var: + if type(k) is dict: delete_nulls(k) + if len(var) == 0: return True + return False + +def get_next_2pow(var): + if var==0: return 0 + v=1 + while v) in case of error""" - warning_text="" - try: - #Get virsh and ssh connection - (return_status, code) = get_ssh_connection(machine, user, password) - if not return_status: - print 'RADclass.obtain_RAD() error:', code - return (return_status, code) - ssh_conn = code - - self.connection_IP = machine - #print "libvirt open pre" - virsh_conn=libvirt.open("qemu+ssh://"+user+'@'+machine+"/system") - #virsh_conn=libvirt.openAuth("qemu+ssh://"+user+'@'+machine+"/system", - # [[libvirt.VIR_CRED_AUTHNAME, libvirt.VIR_CRED_PASSPHRASE, libvirt.VIR_CRED_USERNAME], getCredentials, password], - # 0) - #print "libvirt open after" - - # #Set connection infomation - # (return_status, code) = self.set_connection_info(machine, user, password) - # if not return_status: - # return (return_status, 'Error in '+machine+': '+code) - - #Set server name - machine_name = get_hostname(virsh_conn) - (return_status, code) = self.set_name(machine_name) - if not return_status: - return (return_status, 'Error at self.set_name in '+machine+': '+code) - warning_text += code - - #Get the server processors information - processors = dict() - (return_status, code) = get_processor_information(ssh_conn, virsh_conn, processors) - if not return_status: - return (return_status, 'Error at get_processor_information in '+machine+': '+code) - warning_text += code - - #Get the server memory information - memory_nodes = dict() - (return_status, code) = get_memory_information(ssh_conn, virsh_conn, memory_nodes) - if not return_status: - return (return_status, 'Error at get_memory_information in '+machine+': '+code) - warning_text += code - - #Get nics information - nic_topology = dict() - # (return_status, code) = get_nic_information_old(ssh_conn, nic_topology) - (return_status, code) = get_nic_information(ssh_conn, virsh_conn, nic_topology) - if not return_status: - return (return_status, 'Error at get_nic_information in '+machine+': '+code) - warning_text += code - - #Pack each processor, memory node and nics in a node element - #and add the node to the RAD element - for socket_id, processor in processors.iteritems(): - node = Node() - if not socket_id in nic_topology: - nic_topology[socket_id] = list() - - (return_status, code) = node.set(processor, memory_nodes[socket_id], nic_topology[socket_id]) - # else: - # (return_status, code) = node.set(processor, memory_nodes[socket_id]) - if not return_status: - return (return_status, 'Error at node.set in '+machine+': '+code) - warning_text += code - (return_status, code) = self.insert_node(node) - if not return_status: - return (return_status, 'Error at self.insert_node in '+machine+': '+code) - if code not in warning_text: - warning_text += code - - #Fill os data - os = OpSys() - (return_status, code) = get_os_information(ssh_conn, os) - if not return_status: - return (return_status, 'Error at get_os_information in '+machine+': '+code) - warning_text += code - (return_status, code) = self.set_os(os) - if not return_status: - return (return_status, 'Error at self.set_os in '+machine+': '+code) - warning_text += code - - #Fill hypervisor data - hypervisor = Hypervisor() - (return_status, code) = get_hypervisor_information(virsh_conn, hypervisor) - if not return_status: - return (return_status, 'Error at get_hypervisor_information in '+machine+': '+code) - warning_text += code - (return_status, code) = self.set_hypervisor(hypervisor) - if not return_status: - return (return_status, 'Error at self.set_hypervisor in '+machine+': '+code) - warning_text += code - ssh_conn.close() - - return (True, warning_text) - except libvirt.libvirtError, e: - text = e.get_error_message() - print 'RADclass.obtain_RAD() exception:', text - return (False, text) - except paramiko.ssh_exception.SSHException, e: - text = e.args[0] - print "obtain_RAD ssh Exception:", text - return False, text - - def set_name(self,name): - """Sets the machine name. - Returns (True,Warning) in case of success and ('False',) in case of error""" - if not isinstance(name,str): - return (False, 'The variable \'name\' must be text') - self.name = name - return (True, "") - - def set_connection_info(self, machine, user, password): - """Sets the connection information. - Returns (True,Warning) in case of success and ('False',) in case of error""" - if not isinstance(machine,str): - return (False, 'The variable \'machine\' must be text') - if not isinstance(user,str): - return (False, 'The variable \'user\' must be text') -# if not isinstance(password,str): -# return (False, 'The variable \'password\' must be text') - (self.machine, self.user, self.password) = (machine, user, password) - return (True, "") - - def insert_node(self,node): - """Inserts a new node and updates class variables. - Returns (True,Warning) in case of success and ('False',) in case of error""" - if not isinstance(node,Node): - return (False, 'The variable \'node\' must be a Node element') - - if node.id_ in self.nodes: - return (False, 'The node is already present in the nodes list.') - - #Check if network ports have not been inserted previously as part of another node - for port_key in node.ports_list: - if port_key in self.ports_list: - return (False, 'Network port '+port_key+' defined multiple times in the system') - self.ports_list.append(port_key) - - #Insert the new node - self.nodes[node.id_] = node - - #update variables - self.update_variables() - - return (True, "") - - def update_variables(self): - """Updates class variables. - Returns (True,Warning) in case of success and ('False',) in case of error""" - warning_text="" - #The number of processors and nodes is the same - self.nr_processors = len(self.nodes) - - #If all processors are the same get the values. Otherwise keep them as none - prev_processor_family = prev_processor_manufacturer = prev_processor_version = prev_processor_features = None - different_processor_family = different_processor_manufacturer = different_processor_version = different_processor_features = False - for node in self.nodes.itervalues(): - (self.processor_family, self.processor_manufacturer, self.processor_version, self.processor_features) = node.get_processor_info() - if prev_processor_family != None and self.processor_family != prev_processor_family: - different_processor_family = True - if prev_processor_manufacturer != None and self.processor_manufacturer != prev_processor_manufacturer: - different_processor_manufacturer = True - if prev_processor_version != None and self.processor_version != prev_processor_version: - different_processor_version = True - if prev_processor_features != None and self.processor_features != prev_processor_features: - different_processor_features = True - (prev_processor_family, prev_processor_manufacturer, prev_processor_version, prev_processor_features) = (self.processor_family, self.processor_manufacturer, self.processor_version, self.processor_features) - - if different_processor_family: - self.processor_family = None - if different_processor_features: - self.processor_features = None - if different_processor_manufacturer: - self.processor_manufacturer = None - if different_processor_version: - self.processor_version = None - - #If all memory nodes are the same get the values. Otherwise keep them as none - #Sum the total memory - self.memory_size = 0 - different_memory_freq = different_memory_nr_channels = different_memory_type = different_memory_hugepage_sz = False - prev_memory_freq = prev_memory_nr_channels = prev_memory_type = prev_memory_hugepage_sz = None - for node in self.nodes.itervalues(): - (self.memory_freq, self.memory_nr_channels, self.memory_type, memory_size, self.memory_hugepage_sz) = node.get_memory_info() - self.memory_size += memory_size - if prev_memory_freq != None and self.memory_freq != prev_memory_freq: - different_memory_freq = True - if prev_memory_nr_channels != None and self.memory_nr_channels != prev_memory_nr_channels: - different_memory_nr_channels = True - if prev_memory_type != None and self.memory_type != prev_memory_type: - different_memory_type = True - if prev_memory_hugepage_sz != None and self.memory_hugepage_sz != prev_memory_hugepage_sz: - different_memory_hugepage_sz = True - (prev_memory_freq, prev_memory_nr_channels, prev_memory_type, prev_memory_hugepage_sz) = (self.memory_freq, self.memory_nr_channels, self.memory_type, self.memory_hugepage_sz) - - if different_memory_freq: - self.memory_freq = None - if different_memory_nr_channels: - self.memory_nr_channels = None - if different_memory_type: - self.memory_type = None - if different_memory_hugepage_sz: - warning_text += 'Detected different hugepages size in different sockets\n' - - return (True, warning_text) - - def set_hypervisor(self,hypervisor): - """Sets the hypervisor. - Returns (True,Warning) in case of success and ('False',) in case of error""" - if not isinstance(hypervisor,Hypervisor): - return (False, 'The variable \'hypervisor\' must be of class Hypervisor') - - self.hypervisor.assign(hypervisor) - return (True, "") - - def set_os(self,os): - """Sets the operating system. - Returns (True,Warning) in case of success and ('False',) in case of error""" - if not isinstance(os,OpSys): - return (False, 'The variable \'os\' must be of class OpSys') - - self.os.assign(os) - return (True, "") - - def to_text(self): - text= 'name: '+str(self.name)+'\n' - text+= 'processor:\n' - text+= ' nr_processors: '+str(self.nr_processors)+'\n' - text+= ' family: '+str(self.processor_family)+'\n' - text+= ' manufacturer: '+str(self.processor_manufacturer)+'\n' - text+= ' version: '+str(self.processor_version)+'\n' - text+= ' features: '+str(self.processor_features)+'\n' - text+= 'memory:\n' - text+= ' type: '+str(self.memory_type)+'\n' - text+= ' freq: '+str(self.memory_freq)+'\n' - text+= ' nr_channels: '+str(self.memory_nr_channels)+'\n' - text+= ' size: '+str(self.memory_size)+'\n' - text+= 'hypervisor:\n' - text+= self.hypervisor.to_text() - text+= 'os:\n' - text+= self.os.to_text() - text+= 'resource topology:\n' - text+= ' nr_nodes: '+ str(len(self.nodes))+'\n' - text+= ' nodes:\n' - for node_k, node_v in self.nodes.iteritems(): - text+= ' node'+str(node_k)+':\n' - text+= node_v.to_text() - return text - - def to_yaml(self): - return yaml.load(self.to_text()) - -class Node(): - def __init__(self): - self.id_ = None #Integer. Node id. Unique in the system - self.processor = ProcessorNode() #Information about the processor in the node - self.memory = MemoryNode() #Information about the memory in the node - self.nic_list = list() #List of Nic() containing information about the nics associated to the node - self.ports_list = list() #List containing all network ports in the node. This is used to avoid having defined multiple times the same port in the system - - def get_processor_info(self): - """Gets the processor information. Returns (processor_family, processor_manufacturer, processor_version, processor_features)""" - return self.processor.get_info() - - def get_memory_info(self): - """Gets the memory information. Returns (memory_freq, memory_nr_channels, memory_type, memory_size)""" - return self.memory.get_info() - -# def set(self, *args): -# """Sets the node information. Returns (True,Warning) in case of success and ('False',) in case of error""" -# if len(args)==2: -# processor = args[0] -# memory = args[1] -# nics = False -# elif len(args)==3: -# processor = args[0] -# memory = args[1] -# nic_list = args[2] -# nics = True -# else: -# return (False, 'Wrong number of elements calling Node().set()') - - def set(self, processor, memory, nic_list): - (status, return_code) = self.processor.assign(processor) - if not status: - return (status, return_code) - - self.id_ = processor.id_ - - (status, return_code) = self.memory.assign(memory) - if not status: - return (status, return_code) - -# if nics: - for nic in nic_list: - if not isinstance(nic,Nic): - return (False, 'The nics must be of type Nic') - self.nic_list.append(nic) - for port_key in nic.ports.iterkeys(): - if port_key in self.ports_list: - return (False, 'Network port '+port_key+'defined multiple times in the same node') - self.ports_list.append(port_key) - - return (True,"") - - def assign(self, node): - """Sets the node information. - Returns (True,Warning) in case of success and ('False',) in case of error""" - warning_text="" - processor = node.processor - memory = node.memory - nic_list = node.nic_list - (status, return_code) = self.processor.assign(processor) - if not status: - return (status, return_code) - - self.id_ = processor.id_ - - (status, return_code) = self.memory.assign(memory) - if not status: - return (status, return_code) - warning_text += code - - for nic in nic_list: - if not isinstance(nic,Nic): - return (False, 'The nics must be of type Nic') - self.nic_list.append(nic) - for port_key in nic.ports.iterkeys(): - if port_key in self.ports_list: - return (False, 'Network port '+port_key+'defined multiple times in the same node') - self.ports_list.append(port_key) - - return (True,warning_text) - - def to_text(self): - text= ' id: '+str(self.id_)+'\n' - text+= ' cpu:\n' - text += self.processor.to_text() - text+= ' memory:\n' - text += self.memory.to_text() - if len(self.nic_list) > 0: - text+= ' nics:\n' - nic_index = 0 - for nic in self.nic_list: - text+= ' nic '+str(nic_index)+':\n' - text += nic.to_text() - nic_index += 1 - return text - -class ProcessorNode(): - #Definition of the possible values of processor variables - possible_features = definitionsClass.processor_possible_features - possible_manufacturers = definitionsClass.processor_possible_manufacturers - possible_families = definitionsClass.processor_possible_families - possible_versions = definitionsClass.processor_possible_versions - - def __init__(self): - self.id_ = None #Integer. Numeric identifier of the socket - self.family = None #Text. Family name of the processor - self.manufacturer = None #Text. Manufacturer of the processor - self.version = None #Text. Model version of the processor - self.features = list() #list. List of features offered by the processor - self.cores = list() #list. List of cores in the processor. In case of hyperthreading the coupled cores are expressed as [a,b] - self.eligible_cores = list()#list. List of cores that can be used - #self.decicated_cores - #self.shared_cores -> this should also contain information to know if cores are being used - - def assign(self, processor): - """Sets the processor information. - Returns (True,Warning) in case of success and ('False',) in case of error""" - if not isinstance(processor,ProcessorNode): - return (False, 'The variable \'processor\' must be of class ProcessorNode') - - self.id_ = processor.id_ - self.family = processor.family - self.manufacturer = processor.manufacturer - self.version = processor.version - self.features = processor.features - self.cores = processor.cores - self.eligible_cores = processor.eligible_cores - - return (True, "") - - def set(self, id_, family, manufacturer, version, features, cores): - """Sets the processor information. - Returns (True,Warning) in case of success and ('False',) in case of error""" - warning_text = "" - - if not isinstance(id_,int): - return (False, 'The processor id_ must be of type int') - if not isinstance(family,str): - return (False, 'The processor family must be of type str') - if not isinstance(manufacturer,str): - return (False, 'The processor manufacturer must be of type str') - if not isinstance(version,str): - return (False, 'The processor version must be of type str') - if not isinstance(features,list): - return (False, 'The processor features must be of type list') - if not isinstance(cores,list): - return (False, 'The processor cores must be of type list') - - (self.id_, self.family, self.manufacturer, self.version) = (id_, family, manufacturer, version) - - if not manufacturer in self.possible_manufacturers: - warning_text += "processor manufacturer '%s' not among: %s\n" %(manufacturer, str(self.possible_manufacturers)) - if not family in self.possible_families: - warning_text += "family '%s' not among: %s\n" % (family, str(self.possible_families)) -# if not version in self.possible_versions: -# warning_text += 'The version %s is not one of these: %s\n' % (version, str(self.possible_versions)) - - for feature in features: - if not feature in self.possible_features: - warning_text += "processor feature '%s' not among: %s\n" % (feature, str(self.possible_versions)) - self.features.append(feature) - - for iterator in sorted(cores): - if not isinstance(iterator,list) or not all(isinstance(x, int) for x in iterator): - return (False, 'The cores list must be in the form of [[a,b],[c,d],...] where a,b,c,d are of type int') - self.cores.append(iterator) - - self.set_eligible_cores() - - return (True,warning_text) - - def set_eligible_cores(self): - """Set the default eligible cores, this is all cores non used by the host operating system""" - not_first = False - for iterator in self.cores: - if not_first: - self.eligible_cores.append(iterator) - else: - not_first = True - return - - def get_info(self): - """Returns processor parameters (self.family, self.manufacturer, self.version, self.features)""" - return (self.family, self.manufacturer, self.version, self.features) - - def to_text(self): - text= ' id: '+str(self.id_)+'\n' - text+= ' family: '+self.family+'\n' - text+= ' manufacturer: '+self.manufacturer+'\n' - text+= ' version: '+self.version+'\n' - text+= ' features: '+str(self.features)+'\n' - text+= ' cores: '+str(self.cores)+'\n' - text+= ' eligible_cores: '+str(self.eligible_cores)+'\n' - return text - -class MemoryNode(): - def __init__(self): - self.modules = list() #List of MemoryModule(). List of all modules installed in the node - self.nr_channels = None #Integer. Number of modules installed in the node - self.node_size = None #Integer. Total size in KiB of memory installed in the node - self.eligible_memory = None #Integer. Size in KiB of eligible memory in the node - self.hugepage_sz = None #Integer. Size in KiB of hugepages - self.hugepage_nr = None #Integer. Number of hugepages allocated in the module - self.eligible_hugepage_nr = None #Integer. Number of eligible hugepages in the node - self.type_ = None #Text. Type of memory modules. If modules have a different value keep it as None - self.freq = None #Integer. Frequency of the modules in MHz. If modules have a different value keep it as None - self.module_size = None #Integer. Size of the modules in KiB. If modules have a different value keep it as None - self.form_factor = None #Text. Form factor of the modules. If modules have a different value keep it as None - - def assign(self, memory_node): - return self.set(memory_node.modules, memory_node.hugepage_sz, memory_node.hugepage_nr) - - def set(self, modules, hugepage_sz, hugepage_nr): - """Set the memory node information. hugepage_sz must be expressed in KiB. - Returns (True,Warning) in case of success and ('False',) in case of error""" - if not isinstance(modules, list): - return (False, 'The modules must be a list of elements of class MemoryModule') - if not isinstance(hugepage_sz,int): - return (False, 'The hugepage_sz variable must be an int expressing the size in KiB') - if not isinstance(hugepage_nr,int): - return (False, 'The hugepage_nr variable must be of type int') - - (self.hugepage_sz, self.hugepage_nr) = (hugepage_sz, hugepage_nr) - self.node_size = self.nr_channels = 0 - - different_type = different_freq = different_module_size = different_form_factor = False - prev_type = prev_freq = prev_module_size = prev_form_factor = None - for iterator in modules: - if not isinstance(iterator,MemoryModule): - return (False, 'The modules must be a list of elements of class MemoryModule') - self.modules.append(iterator) - (self.type_, self.freq, self.module_size, self.form_factor) = (iterator.type_, iterator.freq, iterator.size, iterator.form_factor) - self.node_size += self.module_size - self.nr_channels += 1 - if prev_type != None and prev_type != self.type_: - different_type = True - if prev_freq != None and prev_freq != self.freq: - different_freq = True - if prev_module_size != None and prev_module_size != self.module_size: - different_module_size = True - if prev_form_factor != None and prev_form_factor != self.form_factor: - different_form_factor = True - (prev_type, prev_freq, prev_module_size, prev_form_factor) = (self.type_, self.freq, self.module_size, self.form_factor) - - if different_type: - self.type_ = None - if different_freq: - self.freq = None - if different_module_size: - self.module_size = None - if different_form_factor: - self.form_factor = None - - (return_value, error_code) = self.set_eligible_memory() - if not return_value: - return (return_value, error_code) - - return (True, "") - - def set_eligible_memory(self): - """Sets the default eligible_memory and eligible_hugepage_nr. This is all memory but 2GiB and all hugepages""" - self.eligible_memory = self.node_size - 2*1024*1024 - if self.eligible_memory < 0: - return (False, "There is less than 2GiB of memory in the module") - - self.eligible_hugepage_nr = self.hugepage_nr - return (True,"") - - def get_info(self): - """Return memory information (self.freq, self.nr_channels, self.type_, self.node_size)""" - return (self.freq, self.nr_channels, self.type_, self.node_size, self.hugepage_sz) - - def to_text(self): - text= ' node_size: '+str(self.node_size)+'\n' - text+= ' nr_channels: '+str(self.nr_channels)+'\n' - text+= ' eligible_memory: '+str(self.eligible_memory)+'\n' - text+= ' hugepage_sz: '+str(self.hugepage_sz)+'\n' - text+= ' hugepage_nr: '+str(self.hugepage_nr)+'\n' - text+= ' eligible_hugepage_nr: '+str(self.eligible_hugepage_nr)+'\n' - text+= ' type: '+self.type_+'\n' - text+= ' freq: '+str(self.freq)+'\n' - text+= ' module_size: '+str(self.module_size)+'\n' - text+= ' form_factor: '+self.form_factor+'\n' - text+= ' modules details:\n' - for module in self.modules: - text += module.to_text() - return text - -class MemoryModule(): - #Definition of the possible values of module variables - possible_types = definitionsClass.memory_possible_types - possible_form_factors = definitionsClass.memory_possible_form_factors - - def __init__(self): - self.locator = None #Text. Name of the memory module - self.type_ = None #Text. Type of memory module - self.freq = None #Integer. Frequency of the module in MHz - self.size = None #Integer. Size of the module in KiB - self.form_factor = None #Text. Form factor of the module - - def set(self, locator, type_, freq, size, form_factor): - """Sets the memory module information. - Frequency must be expressed in MHz and size in KiB. - Returns (True,Warning) in case of success and ('False',) in case of error""" - warning_text="" - if not isinstance(locator, str): - return (False, "The type of the variable locator must be str") - if not isinstance(type_, str): - return (False, "The type of the variable type_ must be str") - if not isinstance(form_factor, str): - return (False, "The type of the variable form_factor must be str") - if not isinstance(freq, int): - return (False, "The type of the variable freq must be int") - if not isinstance(size, int): - return (False, "The type of the variable size must be int") - - if not form_factor in self.possible_form_factors: - warning_text += "memory form_factor '%s' not among: %s\n" %(form_factor, str(self.possible_form_factors)) - if not type_ in self.possible_types: - warning_text += "memory type '%s' not among: %s\n" %(type_, str(self.possible_types)) - - (self.locator, self.type_, self.freq, self.size, self.form_factor) = (locator, type_, freq, size, form_factor) - return (True, warning_text) - - def to_text(self): - text= ' '+self.locator+':\n' - text+= ' type: '+self.type_+'\n' - text+= ' freq: '+str(self.freq)+'\n' - text+= ' size: '+str(self.size)+'\n' - text+= ' form factor: '+self.form_factor+'\n' - return text - -class Nic(): - def __init__(self): - self.model = None #Text. Model of the nic - self.ports = dict() #Dictionary of ports. Keys are the port name, value are Port() elements - - def set_model(self, model): - """Sets the model of the nic. Returns (True,Warning) in case of success and ('False',) in case of error""" - if not isinstance(model,str): - return (False, 'The \'model\' must be of type str') - - self.model = model - return (True, "") - - def add_port(self, port): - """Adds a port to the nic. Returns (True,Warning) in case of success and ('False',) in case of error""" - if not isinstance(port,Port): - return (False, 'The \'port\' must be of class Port') - -# port_id = str(port.pci_device_id[0])+':'+str(port.pci_device_id[1])+':'+str(port.pci_device_id[2])+'.'+str(port.pci_device_id[3]) -#CHANGED -# port_id = port.name - port_id = port.pci_device_id -#CHANGED END - if port_id in self.ports: - return (False, 'The \'port\' '+port.pci_device_id+' is duplicated in the nic') -# return (False, 'The \'port\' is duplicated in the nic') - - self.ports[port_id] = port - return (True, "") - - def to_text(self): - text= ' model: '+ str(self.model)+'\n' - text+= ' ports: '+'\n' - for key,port in self.ports.iteritems(): - text+= ' "'+key+'":'+'\n' - text += port.to_text() - return text - -class Port(): - def __init__(self): - self.name = None #Text. Port name - self.virtual = None #Boolean. States if the port is a virtual function - self.enabled = None #Boolean. States if the port is enabled - self.eligible = None #Boolean. States if the port is eligible - self.speed = None #Integer. Indicates the speed in Mbps - self.available_bw = None #Integer. BW in Mbps that is available. - self.mac = None #list. Indicates the mac address of the port as a list in format ['XX','XX','XX','XX','XX','XX'] - self.pci_device_id_split = None #list. Indicates the pci address of the port as a list in format ['XXXX','XX','XX','X'] - self.pci_device_id = None - self.PF_pci_device_id = None - -# def set(self, name, virtual, enabled, speed, mac, pci_device_id, pci_device_id_split): -# """Sets the port information. The variable speed indicates the speed in Mbps. Returns (True,Warning) in case of success and ('False',) in case of error""" -# if not isinstance(name,str): -# return (False, 'The variable \'name\' must be of type str') -# if not isinstance(virtual,bool): -# return (False, 'The variable \'virtual\' must be of type bool') -# if not isinstance(enabled,bool): -# return (False, 'The variable \'enabled\' must be of type bool') -# if not isinstance(enabled,bool): -# return (speed, 'The variable \'speed\' must be of type int') -# if not isinstance(mac, list) and not isinstance(mac,NoneType): -# return (False, 'The variable \'enabled\' must be of type list indicating the mac address in format [\'XXXX\',\'XX\',\'XX\',\'X\'] or NoneType') -# if not isinstance(pci_device_id_split, list) or len(pci_device_id_split) != 4: -# return (False, 'The variable \'pci_device_id_split\' must be of type list, indicating the pci address in format [\'XX\',\'XX\',\'XX\',\'XX\',\'XX\',\'XX\']') -# -# expected_len = [4,2,2,1] -# index = 0 -# for iterator in pci_device_id_split: -# if not isinstance(iterator,str) or not iterator.isdigit() or len(iterator) != expected_len[index]: -# return (False, 'The variable \'pci_device_id_split\' must be of type list, indicating the pci address in format [\'XX\',\'XX\',\'XX\',\'XX\',\'XX\',\'XX\']') -# index += 1 -# -# if not isinstance(mac,NoneType): -# for iterator in mac: -# if not isinstance(iterator,str) or not iterator.isalnum() or len(iterator) != 2: -# return (False, 'The variable \'enabled\' must be of type list indicating the mac address in format [\'XXXX\',\'XX\',\'XX\',\'X\'] or NoneType') -# -# #By default only virtual ports are eligible -# # (self.name, self.virtual, self.enabled, self.eligible, self.available_bw, self.speed, self.mac, self.pci_device_id, self.pci_device_id_split) = (name, virtual, enabled, virtual, speed, speed, mac, pci_device_id, pci_device_id_split) -# (self.name, self.virtual, self.enabled, self.eligible, self.available_bw, self.mac, self.pci_device_id, self.pci_device_id_split) = (name, virtual, enabled, virtual, speed, mac, pci_device_id, pci_device_id_split) - - def to_text(self): - text= ' pci: "'+ str(self.pci_device_id)+'"\n' - text+= ' virtual: '+ str(self.virtual)+'\n' - if self.virtual: - text+= ' PF_pci_id: "'+self.PF_pci_device_id+'"\n' - text+= ' eligible: '+ str(self.eligible)+'\n' - text+= ' enabled: '+str(self.enabled)+'\n' - text+= ' speed: '+ str(self.speed)+'\n' - text+= ' available bw: '+ str(self.available_bw)+'\n' - text+= ' mac: '+ str(self.mac)+'\n' - text+= ' source_name: '+ str(self.name)+'\n' - return text - -class Hypervisor(): - #Definition of the possible values of hypervisor variables - possible_types = definitionsClass.hypervisor_possible_types - possible_domain_types = definitionsClass.hypervisor_possible_domain_types - - def __init__(self): - self.type_ = None #Text. Hypervisor type_ - self.version = None #int. Hypervisor version - self.lib_version = None #int. Libvirt version used to compile hypervisor - self.domains = list() #list. List of all the available domains - - def set(self, hypervisor, version, lib_version, domains): - warning_text="" - if not isinstance(hypervisor,str): - return (False, 'The variable type_ must be of type str') - if not isinstance(version,int): - return (False, 'The variable version must be of type int') - if not isinstance(lib_version,int): - return (False, 'The library version must be of type int') - if not isinstance(domains,list): - return (False, 'Domains must be a list of the possible domains as str') - - if not hypervisor in self.possible_types: - warning_text += "Hyperpivor '%s' not among: %s\n" % (hypervisor, str(self.possible_types)) - - valid_domain_found = False - for domain in domains: - if not isinstance(domain,str): - return (False, 'Domains must be a list of the possible domains as str') - if domain in self.possible_domain_types: - valid_domain_found = True - self.domains.append(domain) - - if not valid_domain_found: - warning_text += 'No valid domain found among: %s\n' % str(self.possible_domain_types) - - - (self.version, self.lib_version, self.type_) = (version, lib_version, hypervisor) - return (True, warning_text) - - def assign(self, hypervisor): - (self.version, self.lib_version, self.type_) = (hypervisor.version, hypervisor.lib_version, hypervisor.type_) - for domain in hypervisor.domains: - self.domains.append(domain) - return - - def to_text(self): - text= ' type: '+self.type_+'\n' - text+= ' version: '+str(self.version)+'\n' - text+= ' libvirt version: '+ str(self.lib_version)+'\n' - text+= ' domains: '+str(self.domains)+'\n' - return text - -class OpSys(): - #Definition of the possible values of os variables - possible_id = definitionsClass.os_possible_id - possible_types = definitionsClass.os_possible_types - possible_architectures = definitionsClass.os_possible_architectures - - def __init__(self): - self.id_ = None #Text. Identifier of the OS. Formed by --. In linux this can be obtained using lsb_release -a - self.type_ = None #Text. Type of operating system - self.bit_architecture = None #Integer. Architecture - - def set(self, id_, type_, bit_architecture): - warning_text="" - if not isinstance(type_,str): - return (False, 'The variable type_ must be of type str') - if not isinstance(id_,str): - return (False, 'The variable id_ must be of type str') - if not isinstance(bit_architecture,str): - return (False, 'The variable bit_architecture must be of type str') - - if not type_ in self.possible_types: - warning_text += "os type '%s' not among: %s\n" %(type_, str(self.possible_types)) - if not id_ in self.possible_id: - warning_text += "os release '%s' not among: %s\n" %(id_, str(self.possible_id)) - if not bit_architecture in self.possible_architectures: - warning_text += "os bit_architecture '%s' not among: %s\n" % (bit_architecture, str(self.possible_architectures)) - - (self.id_, self.type_, self.bit_architecture) = (id_, type_, bit_architecture) - return (True, warning_text) - - def assign(self,os): - (self.id_, self.type_, self.bit_architecture) = (os.id_, os.type_, os.bit_architecture) - return - - def to_text(self): - text= ' id: '+self.id_+'\n' - text+= ' type: '+self.type_+'\n' - text+= ' bit_architecture: '+self.bit_architecture+'\n' - return text - -def get_hostname(virsh_conn): - return virsh_conn.getHostname().rstrip('\n') - -def get_hugepage_size(ssh_conn): - command = 'sudo hugeadm --page-sizes' -# command = 'hugeadm --page-sizes-all' - (_, stdout, stderr) = ssh_conn.exec_command(command) - error = stderr.read() - if len(error)>0: - raise paramiko.ssh_exception.SSHException(command +' : '+ error) - mem=stdout.read() - if mem=="": - return 0 - return int(mem) - -def get_hugepage_nr(ssh_conn,hugepage_sz, node_id): - command = 'cat /sys/devices/system/node/node'+str(node_id)+'/hugepages/hugepages-'+str(hugepage_sz/1024)+'kB/nr_hugepages' - (_, stdout, _) = ssh_conn.exec_command(command) - #print command, - #text = stdout.read() - #print "'"+text+"'" - #return int(text) - - try: - value=int(stdout.read()) - except: - value=0 - return value - -def get_memory_information(ssh_conn, virsh_conn, memory_nodes): - warning_text="" - tree=ElementTree.fromstring(virsh_conn.getSysinfo(0)) - memory_dict = dict() - node_id = 0 #TODO revise. Added for allowing VM as compute hosts - for target in tree.findall("memory_device"): - locator_f = size_f = freq_f = type_f = formfactor_f = False - locator_f = True #TODO revise. Added for allowing VM as compute hosts - module_form_factor = "" - for entry in target.findall("entry"): - if entry.get("name") == 'size': - size_f = True - size_split = entry.text.split(' ') - if size_split[1] == 'MB': - module_size = int(size_split[0]) * 1024 * 1024 - elif size_split[1] == 'GB': - module_size = int(size_split[0]) * 1024 * 1024 * 1024 - elif size_split[1] == 'KB': - module_size = int(size_split[0]) * 1024 - else: - module_size = int(size_split[0]) - - elif entry.get("name") == 'speed': - freq_f = True - freq_split = entry.text.split(' ') - if freq_split[1] == 'MHz': - module_freq = int(freq_split[0]) * 1024 * 1024 - elif freq_split[1] == 'GHz': - module_freq = int(freq_split[0]) * 1024 * 1024 * 1024 - elif freq_split[1] == 'KHz': - module_freq = int(freq_split[0]) * 1024 - - elif entry.get("name") == 'type': - type_f = True - module_type = entry.text - - elif entry.get("name") == 'form_factor': - formfactor_f = True - module_form_factor = entry.text - #TODO revise. Commented for allowing VM as compute hosts - # elif entry.get("name") == 'locator' and not locator_f: - # # other case, it is obtained by bank_locator that we give priority to - # locator = entry.text - # pos = locator.find(module_form_factor) - # if module_form_factor == locator[0:len(module_form_factor) ]: - # pos = len(module_form_factor) +1 - # else: - # pos = 0 - # if locator[pos] in "ABCDEFGH": - # locator_f = True - # node_id = ord(locator[pos])-ord('A') - # #print entry.text, node_id - # - # elif entry.get("name") == 'bank_locator': - # locator = entry.text - # pos = locator.find("NODE ") - # if pos >= 0 and len(locator)>pos+5: - # if locator[pos+5] in ("01234567"): #len("NODE ") is 5 - # node_id = int(locator[pos+5]) - # locator_f = True - # - - #When all module fields have been found add a new module to the list - if locator_f and size_f and freq_f and type_f and formfactor_f: - #If the memory node has not yet been created create it - if node_id not in memory_dict: - memory_dict[node_id] = [] - - #Add a new module to the memory node - module = MemoryModule() - #TODO revise. Changed for allowing VM as compute hosts - (return_status, code) = module.set('NODE %d' % node_id, module_type, module_freq, module_size, module_form_factor) - #(return_status, code) = module.set(locator, module_type, module_freq, module_size, module_form_factor) - if not return_status: - return (return_status, code) - memory_dict[node_id].append(module) - if code not in warning_text: - warning_text += code - node_id += 1 #TODO revise. Added for allowing VM as compute hosts - - #Fill memory nodes - #Hugepage size is constant for all nodes - hugepage_sz = get_hugepage_size(ssh_conn) - for node_id, modules in memory_dict.iteritems(): - memory_node = MemoryNode() - memory_node.set(modules, hugepage_sz, get_hugepage_nr(ssh_conn,hugepage_sz, node_id)) - memory_nodes[node_id] = memory_node - - return (True, warning_text) - -def get_cpu_topology_ht(ssh_conn, topology): - command = 'cat /proc/cpuinfo' - (_, stdout, stderr) = ssh_conn.exec_command(command) - error = stderr.read() - if len(error)>0: - raise paramiko.ssh_exception.SSHException(command +' : '+ error) - sockets = [] - cores = [] - core_map = {} - core_details = [] - core_lines = {} - for line in stdout.readlines(): - if len(line.strip()) != 0: - name, value = line.split(":", 1) - core_lines[name.strip()] = value.strip() - else: - core_details.append(core_lines) - core_lines = {} - - for core in core_details: - for field in ["processor", "core id", "physical id"]: - if field not in core: - return(False,'Error getting '+field+' value from /proc/cpuinfo') - core[field] = int(core[field]) - - if core["core id"] not in cores: - cores.append(core["core id"]) - if core["physical id"] not in sockets: - sockets.append(core["physical id"]) - key = (core["physical id"], core["core id"]) - if key not in core_map: - core_map[key] = [] - core_map[key].append(core["processor"]) - - for s in sockets: - hyperthreaded_cores = list() - for c in cores: - hyperthreaded_cores.append(core_map[(s,c)]) - topology[s] = hyperthreaded_cores - - return (True, "") - -def get_processor_information(ssh_conn, vish_conn, processors): - warning_text="" - #Processor features are the same for all processors - #TODO (at least using virsh capabilities)nr_numa_nodes - capabilities = list() - tree=ElementTree.fromstring(vish_conn.getCapabilities()) - for target in tree.findall("host/cpu/feature"): - if target.get("name") == 'pdpe1gb': - capabilities.append('lps') - elif target.get("name") == 'dca': - capabilities.append('dioc') - elif target.get("name") == 'vmx' or target.get("name") == 'svm': - capabilities.append('hwsv') - elif target.get("name") == 'ht': - capabilities.append('ht') - - target = tree.find("host/cpu/arch") - if target.text == 'x86_64' or target.text == 'amd64': - capabilities.append('64b') - - command = 'cat /proc/cpuinfo | grep flags' - (_, stdout, stderr) = ssh_conn.exec_command(command) - error = stderr.read() - if len(error)>0: - raise paramiko.ssh_exception.SSHException(command +' : '+ error) - line = stdout.readline() - if 'ept' in line or 'npt' in line: - capabilities.append('tlbps') - - #Find out if IOMMU is enabled - command = 'dmesg |grep -e Intel-IOMMU' - (_, stdout, stderr) = ssh_conn.exec_command(command) - error = stderr.read() - if len(error)>0: - raise paramiko.ssh_exception.SSHException(command +' : '+ error) - if 'enabled' in stdout.read(): - capabilities.append('iommu') - - #Equivalent for AMD - command = 'dmesg |grep -e AMD-Vi' - (_, stdout, stderr) = ssh_conn.exec_command(command) - error = stderr.read() - if len(error)>0: - raise paramiko.ssh_exception.SSHException(command +' : '+ error) - if len(stdout.read()) > 0: - capabilities.append('iommu') - - #----------------------------------------------------------- - topology = dict() - #In case hyperthreading is active it is necessary to determine cpu topology using /proc/cpuinfo - if 'ht' in capabilities: - (return_status, code) = get_cpu_topology_ht(ssh_conn, topology) - if not return_status: - return (return_status, code) - warning_text += code - - #Otherwise it is possible to do it using virsh capabilities - else: - for target in tree.findall("host/topology/cells/cell"): - socket_id = int(target.get("id")) - topology[socket_id] = list() - for cpu in target.findall("cpus/cpu"): - topology[socket_id].append(int(cpu.get("id"))) - - #----------------------------------------------------------- - #Create a dictionary with the information of all processors - #p_fam = p_man = p_ver = None - tree=ElementTree.fromstring(vish_conn.getSysinfo(0)) - #print vish_conn.getSysinfo(0) - #return (False, 'forces error for debuging') - not_populated=False - socket_id = -1 #in case we can not determine the socket_id we assume incremental order, starting by 0 - for target in tree.findall("processor"): - count = 0 - socket_id += 1 - #Get processor id, family, manufacturer and version - for entry in target.findall("entry"): - if entry.get("name") == "status": - if entry.text[0:11] == "Unpopulated": - not_populated=True - elif entry.get("name") == 'socket_destination': - socket_text = entry.text - if socket_text.startswith('CPU'): - socket_text = socket_text.strip('CPU') - socket_text = socket_text.strip() #removes trailing spaces - if socket_text.isdigit() and int(socket_text)<9 and int(socket_text)>0: - socket_id = int(socket_text) - 1 - - elif entry.get("name") == 'family': - family = entry.text - count += 1 - elif entry.get("name") == 'manufacturer': - manufacturer = entry.text - count += 1 - elif entry.get("name") == 'version': - version = entry.text.strip() - count += 1 - if count != 3: - return (False, 'Error. Not all expected fields could be found in processor') - - #Create and fill processor structure - if not_populated: - continue #avoid inconsistence of some machines where more socket detected than - processor = ProcessorNode() - (return_status, code) = processor.set(socket_id, family, manufacturer, version, capabilities, topology[socket_id]) - if not return_status: - return (return_status, code) - if code not in warning_text: - warning_text += code - - #Add processor to the processors dictionary - processors[socket_id] = processor - - return (True, warning_text) - -def get_nic_information(ssh_conn, virsh_conn, nic_topology): - warning_text="" - #Get list of net devices - net_devices = virsh_conn.listDevices('net',0) - print virsh_conn.listDevices('net',0) - for device in net_devices: - try: - #Get the XML descriptor of the device: - net_XML = ElementTree.fromstring(virsh_conn.nodeDeviceLookupByName(device).XMLDesc(0)) - #print "net_XML:" , net_XML - #obtain the parent - parent = net_XML.find('parent') - if parent == None: - print 'No parent was found in XML for device '+device - #Error. continue?------------------------------------------------------------- - continue - if parent.text == 'computer': - continue - if not parent.text.startswith('pci_'): - print device + ' parent is neither computer nor pci' - #Error. continue?------------------------------------------------------------- - continue - interface = net_XML.find('capability/interface').text - mac = net_XML.find('capability/address').text - - #Get the pci XML - pci_XML = ElementTree.fromstring(virsh_conn.nodeDeviceLookupByName(parent.text).XMLDesc(0)) - #print pci_XML - #Get pci - name = pci_XML.find('name').text.split('_') - pci = name[1]+':'+name[2]+':'+name[3]+'.'+name[4] - - #If slot == 0 it is a PF, otherwise it is a VF - capability = pci_XML.find('capability') - if capability.get('type') != 'pci': - print device + 'Capability is not of type pci in '+parent.text - #Error. continue?------------------------------------------------------------- - continue - slot = capability.find('slot').text - bus = capability.find('bus').text - node_id = None - numa_ = capability.find('numa') - if numa_ != None: - node_id = numa_.get('node'); - if node_id != None: node_id =int(node_id) - if slot == None or bus == None: - print device + 'Bus and slot not detected in '+parent.text - #Error. continue?------------------------------------------------------------- - continue - if slot != '0': - # print ElementTree.tostring(pci_XML) - virtual = True - capability_pf = capability.find('capability') - if capability_pf.get('type') != 'phys_function': - print 'physical_function not found in VF '+parent.text - #Error. continue?------------------------------------------------------------- - continue - PF_pci = capability_pf.find('address').attrib - PF_pci_text = PF_pci['domain'].split('x')[1]+':'+PF_pci['bus'].split('x')[1]+':'+PF_pci['slot'].split('x')[1]+'.'+PF_pci['function'].split('x')[1] - - else: - virtual = False - - #Obtain node for the port - if node_id == None: - node_id = int(bus)>>6 - #print "node_id:", node_id - - #Only for non virtual interfaces: Obtain speed and if link is detected (this must be done using ethtool) - if not virtual: - command = 'sudo ethtool '+interface+' | grep -e Speed -e "Link detected"' - (_, stdout, stderr) = ssh_conn.exec_command(command) - error = stderr.read() - if len(error) >0: - print 'Error running '+command+'\n'+error - #Error. continue?------------------------------------------------------------- - continue - for line in stdout.readlines(): - line = line.strip().rstrip('\n').split(': ') - if line[0] == 'Speed': - if line[1].endswith('Mb/s'): - speed = int(line[1].split('M')[0])*int(1e6) - elif line[1].endswith('Gb/s'): - speed = int(line[1].split('G')[0])*int(1e9) - elif line[1].endswith('Kb/s'): - speed = int(line[1].split('K')[0])*int(1e3) - else: - #the interface is listed but won't be used - speed = 0 - elif line[0] == 'Link detected': - if line[1] == 'yes': - enabled = True - else: - enabled = False - else: - print 'Unnexpected output of command '+command+':' - print line - #Error. continue?------------------------------------------------------------- - continue - - if not node_id in nic_topology: - nic_topology[node_id] = list() - #With this implementation we make the RAD with only one nic per node and this nic has all ports, TODO: change this by including parent information of PF - nic_topology[node_id].append(Nic()) - - #Load the appropriate nic - nic = nic_topology[node_id][0] - - #Create a new port and fill it - port = Port() - port.name = interface - port.virtual = virtual - port.speed = speed - if virtual: - port.available_bw = 0 - port.PF_pci_device_id = PF_pci_text - else: - port.available_bw = speed - if speed == 0: - port.enabled = False - else: - port.enabled = enabled - - port.eligible = virtual #Only virtual ports are eligible - port.mac = mac - port.pci_device_id = pci - port.pci_device_id_split = name[1:] - - #Save the port information - nic.add_port(port) - except Exception,e: - print 'Error: '+str(e) - - #set in vitual ports if they are enabled - for nic in nic_topology.itervalues(): - for port in nic[0].ports.itervalues(): -# print port.pci_device_id - if port.virtual: - enabled = nic[0].ports.get(port.PF_pci_device_id) - if enabled == None: - return(False, 'The PF '+port.PF_pci_device_id+' (VF '+port.pci_device_id+') is not present in ports dict') - #Only if the PF is enabled the VF can be enabled - if nic[0].ports[port.PF_pci_device_id].enabled: - port.enabled = True - else: - port.enabled = False - - return (True, warning_text) - -def get_nic_information_old(ssh_conn, nic_topology): - command = 'lstopo-no-graphics --of xml' - (_, stdout, stderr) = ssh_conn.exec_command(command) - error = stderr.read() - if len(error)>0: - raise paramiko.ssh_exception.SSHException(command +' : '+ error) - tree=ElementTree.fromstring(stdout.read()) - for target in tree.findall("object/object"): - #Find numa nodes - if target.get("type") != "NUMANode": - continue - node_id = int(target.get("os_index")) - nic_topology[node_id] = list() - - #find nics in numa node - for entry in target.findall("object/object"): - if entry.get("type") != 'Bridge': - continue - nic_name = entry.get("name") - model = None - nic = Nic() - - #find ports in nic - for pcidev in entry.findall("object"): - if pcidev.get("type") != 'PCIDev': - continue - enabled = speed = mac = pci_busid = None - port = Port() - model = pcidev.get("name") - virtual = False - if 'Virtual' in model: - virtual = True - pci_busid = pcidev.get("pci_busid") - for osdev in pcidev.findall("object"): - name = osdev.get("name") - for info in osdev.findall("info"): - if info.get("name") != 'Address': - continue - mac = info.get("value") - #get the port speed and status - command = 'sudo ethtool '+name - (_, stdout, stderr) = ssh_conn.exec_command(command) - error = stderr.read() - if len(error)>0: - return (False, 'Error obtaining '+name+' information: '+error) - ethtool = stdout.read() - if '10000baseT/Full' in ethtool: - speed = 10e9 - elif '1000baseT/Full' in ethtool: - speed = 1e9 - elif '100baseT/Full' in ethtool: - speed = 100e6 - elif '10baseT/Full' in ethtool: - speed = 10e6 - else: - return (False, 'Speed not detected in '+name) - - enabled = False - if 'Link detected: yes' in ethtool: - enabled = True - - if speed != None and mac != None and pci_busid != None: - mac = mac.split(':') - pci_busid_split = re.split(':|\.', pci_busid) - #Fill the port information - port.set(name, virtual, enabled, speed, mac, pci_busid, pci_busid_split) - nic.add_port(port) - - if len(nic.ports) > 0: - #Fill the nic model - if model != None: - nic.set_model(model) - else: - nic.set_model(nic_name) - - #Add it to the topology - nic_topology[node_id].append(nic) - - return (True, "") - -def get_os_information(ssh_conn, os): - warning_text="" -# command = 'lsb_release -a' -# (stdin, stdout, stderr) = ssh_conn.exec_command(command) -# cont = 0 -# for line in stdout.readlines(): -# line_split = re.split('\t| *', line.rstrip('\n')) -# if line_split[0] == 'Distributor' and line_split[1] == 'ID:': -# distributor = line_split[2] -# cont += 1 -# elif line_split[0] == 'Release:': -# release = line_split[1] -# cont += 1 -# elif line_split[0] == 'Codename:': -# codename = line_split[1] -# cont += 1 -# if cont != 3: -# return (False, 'It was not possible to obtain the OS id') -# id_ = distributor+'-'+release+'-'+codename - - - command = 'cat /etc/redhat-release' - (_, stdout, _) = ssh_conn.exec_command(command) - id_text= stdout.read() - if len(id_text)==0: - #try with Ubuntu - command = 'lsb_release -d -s' - (_, stdout, _) = ssh_conn.exec_command(command) - id_text= stdout.read() - if len(id_text)==0: - raise paramiko.ssh_exception.SSHException("Can not determinte release neither with 'lsb_release' nor with 'cat /etc/redhat-release'") - id_ = id_text.rstrip('\n') - - command = 'uname -o' - (_, stdout, stderr) = ssh_conn.exec_command(command) - error = stderr.read() - if len(error)>0: - raise paramiko.ssh_exception.SSHException(command +' : '+ error) - type_ = stdout.read().rstrip('\n') - - command = 'uname -i' - (_, stdout, stderr) = ssh_conn.exec_command(command) - error = stderr.read() - if len(error)>0: - raise paramiko.ssh_exception.SSHException(command +' : '+ error) - bit_architecture = stdout.read().rstrip('\n') - - (return_status, code) = os.set(id_, type_, bit_architecture) - if not return_status: - return (return_status, code) - warning_text += code - return (True, warning_text) - -def get_hypervisor_information(virsh_conn, hypervisor): - type_ = virsh_conn.getType().rstrip('\n') - version = virsh_conn.getVersion() - lib_version = virsh_conn.getLibVersion() - - domains = list() - tree=ElementTree.fromstring(virsh_conn.getCapabilities()) - for target in tree.findall("guest"): - os_type = target.find("os_type").text - #We only allow full virtualization - if os_type != 'hvm': - continue - wordsize = int(target.find('arch/wordsize').text) - if wordsize == 64: - for domain in target.findall("arch/domain"): - domains.append(domain.get("type")) - - (return_status, code) = hypervisor.set(type_, version, lib_version, domains) - if not return_status: - return (return_status, code) - return (True, code) - -class RADavailableResourcesClass(RADclass): - def __init__(self, resources): - """Copy resources from the RADclass (server resources not taking into account resources used by VMs""" - #New - self.reserved = dict() #Dictionary of reserved resources for a server. Key are VNFC names and values RADreservedResources - self.cores_consumption = None #Dictionary of cpu consumption. Key is the cpu and the value is - - self.machine = resources.machine - self.user = resources.user - self.password = resources.password - self.name = resources.name - self.nr_processors = resources.nr_processors - self.processor_family = resources.processor_family - self.processor_manufacturer = resources.processor_manufacturer - self.processor_version = resources.processor_version - self.processor_features = resources.processor_features - self.memory_type = resources.memory_type - self.memory_freq = resources.memory_freq - self.memory_nr_channels = resources.memory_nr_channels - self.memory_size = resources.memory_size - self.memory_hugepage_sz = resources.memory_hugepage_sz - self.hypervisor = Hypervisor() - self.hypervisor.assign(resources.hypervisor) - self.os = OpSys() - self.os.assign(resources.os) - self.nodes = dict() - for node_k, node_v in resources.nodes.iteritems(): - self.nodes[node_k] = Node() - self.nodes[node_k].assign(node_v) - return - - def _get_cores_consumption_warnings(self): - """Returns list of warning strings in case warnings are generated. - In case no warnings are generated the return value will be an empty list""" - warnings = list() - #Get the cores consumption - (return_status, code) = get_ssh_connection(self.machine, self.user, self.password) - if not return_status: - return (return_status, code) - ssh_conn = code - command = 'mpstat -P ALL 1 1 | grep Average | egrep -v CPU\|all' - (_, stdout, stderr) = ssh_conn.exec_command(command) - error = stderr.read() - if len(error) > 0: - return (False, error) - - self.cores_consumption = dict() - for line in stdout.readlines(): - cpu_usage_split = re.split('\t| *', line.rstrip('\n')) - usage = 100 *(1 - float(cpu_usage_split[10])) - if usage > 0: - self.cores_consumption[int(cpu_usage_split[1])] = usage - ssh_conn.close() - #Check if any core marked as available in the nodes has cpu_usage > 0 - for _, node_v in self.nodes.iteritems(): - cores = node_v.processor.eligible_cores - for cpu in cores: - if len(cpu) > 1: - for core in cpu: - if core in self.cores_consumption: - warnings.append('Warning: Core '+str(core)+' is supposed to be idle but it is consuming '+str(self.cores_consumption[core])+'%') - else: - if cpu in self.cores_consumption: - warnings.append('Warning: Core '+str(core)+' is supposed to be idle but it is consuming '+str(self.cores_consumption[cpu])+'%') - - return warnings - - def reserved_to_text(self): - text = str() - for VNFC_name, VNFC_reserved in self.reserved.iteritems(): - text += ' VNFC: '+str(VNFC_name)+'\n' - text += VNFC_reserved.to_text() - - return text - - def obtain_usage(self): - resp = dict() - #Iterate through nodes to get cores, eligible cores, memory and physical ports (save ports usage for next section) - nodes = dict() - ports_usage = dict() - hugepage_size = dict() - for node_k, node_v in self.nodes.iteritems(): - node = dict() - ports_usage[node_k] = dict() - eligible_cores = list() - for pair in node_v.processor.eligible_cores: - if isinstance(pair, list): - for element in pair: - eligible_cores.append(element) - else: - eligible_cores.append(pair) - node['cpus'] = {'cores':node_v.processor.cores,'eligible_cores':eligible_cores} - node['memory'] = {'size':str(node_v.memory.node_size/(1024*1024*1024))+'GB','eligible':str(node_v.memory.eligible_memory/(1024*1024*1024))+'GB'} - hugepage_size[node_k] = node_v.memory.hugepage_sz - - ports = dict() - for nic in node_v.nic_list: - for port in nic.ports.itervalues(): - if port.enabled and not port.virtual: - ports[port.name] = {'speed':str(port.speed/1000000000)+'G'} -# print '*************** ',port.name,'speed',port.speed - ports_usage[node_k][port.name] = 100 - int(100*float(port.available_bw)/float(port.speed)) - node['ports'] = ports - nodes[node_k] = node - resp['RAD'] = nodes - - #Iterate through reserved section to get used cores, used memory and port usage - cores = dict() - memory = dict() - #reserved_cores = list - for node_k in self.nodes.iterkeys(): - if not node_k in cores: - cores[node_k] = list() - memory[node_k] = 0 - for _, reserved in self.reserved.iteritems(): - if node_k in reserved.node_reserved_resources: - node_v = reserved.node_reserved_resources[node_k] - cores[node_k].extend(node_v.reserved_cores) - memory[node_k] += node_v.reserved_hugepage_nr * hugepage_size[node_k] - - occupation = dict() - for node_k in self.nodes.iterkeys(): - ports = dict() - for name, usage in ports_usage[node_k].iteritems(): - ports[name] = {'occupied':str(usage)+'%'} -# print '****************cores',cores -# print '****************memory',memory - occupation[node_k] = {'cores':cores[node_k],'memory':str(memory[node_k]/(1024*1024*1024))+'GB','ports':ports} - resp['occupation'] = occupation - - return resp - -class RADreservedResources(): - def __init__(self): - self.node_reserved_resources = dict() #dict. keys are the RAD nodes id, values are NodeReservedResources - self.mgmt_interface_pci = None #pci in the VNF for the management interface - self.image = None #Path in remote machine of the VNFC image - - def update(self,reserved): - self.image = reserved.image - self.mgmt_interface_pci = reserved.mgmt_interface_pci - for k,v in reserved.node_reserved_resources.iteritems(): - if k in self.node_reserved_resources.keys(): - return (False, 'Duplicated node entry '+str(k)+' in reserved resources') - self.node_reserved_resources[k]=v - - return (True, "") - - def to_text(self): - text = ' image: '+str(self.image)+'\n' - for node_id, node_reserved in self.node_reserved_resources.iteritems(): - text += ' Node ID: '+str(node_id)+'\n' - text += node_reserved.to_text() - return text - -class NodeReservedResources(): - def __init__(self): - # reserved_shared_cores = None #list. List of all cores that the VNFC needs in shared mode #TODO Not used - # reserved_memory = None #Integer. Amount of KiB needed by the VNFC #TODO. Not used since hugepages are used - self.reserved_cores = list() #list. List of all cores that the VNFC uses - self.reserved_hugepage_nr = 0 #Integer. Number of hugepages needed by the VNFC - self.reserved_ports = dict() #dict. The key is the physical port pci and the value the VNFC port description - self.vlan_tags = dict() - self.cpu_pinning = None - - def to_text(self): - text = ' cores: '+str(self.reserved_cores)+'\n' - text += ' cpu_pinning: '+str(self.cpu_pinning)+'\n' - text += ' hugepages_nr: '+str(self.reserved_hugepage_nr)+'\n' - for port_pci, port_description in self.reserved_ports.iteritems(): - text += ' port: '+str(port_pci)+'\n' - text += port_description.to_text() - return text - -# def update(self,reserved): -# self.reserved_cores = list(reserved.reserved_cores) -# self.reserved_hugepage_nr = reserved.reserved_hugepage_nr -# self.reserved_ports = dict(reserved.reserved_ports) -# self.cpu_pinning = list(reserved.cpu_pinning) - - - diff --git a/utils/auxiliary_functions.py b/utils/auxiliary_functions.py deleted file mode 100644 index 44e7fcb..0000000 --- a/utils/auxiliary_functions.py +++ /dev/null @@ -1,227 +0,0 @@ -# -*- coding: utf-8 -*- - -## -# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. -# This file is part of openmano -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# For those usages not covered by the Apache License, Version 2.0 please -# contact with: nfvlabs@tid.es -## - -''' -Common usuful functions -''' - -__author__="Alfonso Tierno, Pablo Montes" -__date__ ="$10-jul-2014 12:07:15$" - - -import yaml -import paramiko -from definitionsClass import definitionsClass -from definitionsClass import Units -import random -from jsonschema import validate as js_v, exceptions as js_e - -def check_and_convert_units(value, value_type): - """TODO: Update description - This method receives a text with 2 fields using a blank as separator and a list of valid units. The first field must represent a number - and the second one units. - In case the second field is not one of valid_units (False, ) is returned. - In case the second field is a valid unit the first number is converted in the following way: - Gbps, Mbps, kbps -> Mbps - GB,MB,KB,B,GiB,MiB,KiB -> B - GHz,MHz,KHz,Hz -> Hz - If conversion is done successfully (True, ) is returned""" - try: - if value_type == Units.no_units: - if not isinstance(value,int) and not isinstance(value,float): - return (False, 'When no units are used only an integer or float must be used') - elif value_type == Units.name: - if not isinstance(value,str): - return (False, 'For names str must be used') - elif value_type == Units.boolean: - if not isinstance(value,bool): - return (False, 'A boolean or Yes/No mut be used') - else: - splitted = value.split(' ') - if len(splitted) != 2: - return (False, 'Expected format: ') - (value, units) = splitted - if ',' in value or '.' in value: - return (False, 'Use integers to represent numeric values') - - value = int(value) - -# if not isinstance(value_type, Units): -# return (False, 'Not valid value_type') - - valid_units = definitionsClass.units[value_type] - - #Convert everything to upper in order to make comparations easier - units = units.upper() - for i in range(0, len(valid_units)): - valid_units[i] = valid_units[i].upper() - - #Check the used units are valid ones - if units not in valid_units: - return (False, 'Valid units are: '+', '.join(valid_units)) - - if units.startswith('GI'): - value = value *1024*1024*1024 - elif units.startswith('MI'): - value = value *1024*1024 - elif units.startswith('KI'): - value = value *1024 - elif units.startswith('G'): - value = value *1000000000 - elif units.startswith('M'): - value = value *1000000 - elif units.startswith('K'): - value = value *1000 - except Exception,e: - return (False, 'Unexpected error in auxiliary_functions.py - check_and_convert_units:\n'+str(e)) - - return (True, value) - -def get_ssh_connection(machine, user=None, password=None): - """Stablishes an ssh connection to the remote server. Returns (True, paramiko_ssh) in case of success or (False, ) in case of error""" - try: - s = paramiko.SSHClient() - s.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - s.load_system_host_keys() - s.connect(machine, 22, user, password, timeout=10) - except Exception,e: - return (False, 'It was not possible to connect to '+machine+str(e)) - - return (True, s) - -def run_in_remote_server(s,command): - """Runs in the remote server the specified command. Returns (True, stdout) in case of success or (False, ) in case of error""" - try: - (_, stdout, stderr) = s.exec_command(command) - error_msg = stderr.read() - if len(error_msg) > 0: - return (False, error_msg) - except Exception,e: - return (False, str(e)) - - return (True, stdout) - -def read_file(file_): - """Reads a file specified by 'file' and returns (True,) in case of success or (False, ) in case of failure""" - try: - f = open(file_, 'r') - read_data = f.read() - f.close() - except Exception,e: - return (False, str(e)) - - return (True, read_data) - -def check_contains(element, keywords): - """Auxiliary function used to check if a yaml structure contains or not - an specific field. Returns a bool""" - for key in keywords: - if not key in element: - return False - return True - -def check_contains_(element, keywords): - """Auxiliary function used to check if a yaml structure contains or not - an specific field. Returns a bool,missing_variables""" - for key in keywords: - if not key in element: - return False, key - return True, None - -def write_file(file_, content): - """Generates a file specified by 'file' and fills it using 'content'""" - f = open(file_, 'w') - f.write(content) - f.close() - -def nice_print(yaml_element): - """Print a yaml structure. Used mainly for debugging""" - print(yaml.dump(yaml_element, default_flow_style=False)) - -def new_random_mac(): - mac = (0xE2, random.randint(0x00, 0xff), random.randint(0x00, 0xff), random.randint(0x00, 0xff), random.randint(0x00, 0xff), random.randint(0x00, 0xff) ) - return ':'.join(map(lambda x: "%02X" % x, mac)) - -def parse_dict(var, template): - if type(var) is not dict: return -1, 'not a dictionary' - for _,tv in template.items(): - if type(tv) is list: - return - -def delete_nulls(var): - if type(var) is dict: - for k in var.keys(): - if var[k] is None: del var[k] - elif type(var[k]) is dict or type(var[k]) is list or type(var[k]) is tuple: - if delete_nulls(var[k]): del var[k] - if len(var) == 0: return True - elif type(var) is list or type(var) is tuple: - for k in var: - if type(k) is dict: delete_nulls(k) - if len(var) == 0: return True - return False - -def get_next_2pow(var): - if var==0: return 0 - v=1 - while v