+++ /dev/null
-# -*- coding: utf-8 -*-
-
-##
-# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
-# This file is part of openvim
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-#
-# For those usages not covered by the Apache License, Version 2.0 please
-# contact with: nfvlabs@tid.es
-##
-
-'''
-Implement the logic for obtaining compute nodes information
-Resource Availability Descriptor
-'''
-__author__="Pablo Montes"
-
-#TODO: remove warnings, remove unused things
-
-from definitionsClass import definitionsClass
-from auxiliary_functions import get_ssh_connection
-import libvirt
-from xml.etree import ElementTree
-import paramiko
-import re
-import yaml
-
-
-def getCredentials(creds, data):
- """Used as a backup for libvirt.openAuth in order to provide password that came with data,
- not used by the moment
- """
- print "RADclass:getCredentials", creds, data
- for cred in creds:
- print cred[1] + ": ",
- if cred[0] == libvirt.VIR_CRED_AUTHNAME:
- cred[4] = data
- elif cred[0] == libvirt.VIR_CRED_PASSPHRASE:
- cred[4] = data
- else:
- return -1
- return 0
-
-class RADclass():
- def __init__(self):
- self.name = None
- self.machine = None
- self.user = None
- self.password = None
- self.nodes = dict() #Dictionary of nodes. Keys are the node id, values are Node() elements
- self.nr_processors = None #Integer. Number of processors in the system
- self.processor_family = None #If all nodes have the same value equal them, otherwise keep as None
- self.processor_manufacturer = None #If all nodes have the same value equal them, otherwise keep as None
- self.processor_version = None #If all nodes have the same value equal them, otherwise keep as None
- self.processor_features = None #If all nodes have the same value equal them, otherwise keep as None
- self.memory_type = None #If all nodes have the same value equal them, otherwise keep as None
- self.memory_freq = None #If all nodes have the same value equal them, otherwise keep as None
- self.memory_nr_channels = None #If all nodes have the same value equal them, otherwise keep as None
- self.memory_size = None #Integer. Sum of the memory in all nodes
- self.memory_hugepage_sz = None
- self.hypervisor = Hypervisor() #Hypervisor information
- self.os = OpSys() #Operating system information
- self.ports_list = list() #List containing all network ports in the node. This is used to avoid having defined multiple times the same port in the system
-
-
- def obtain_RAD(self, user, password, machine):
- """This function obtains the RAD information from the remote server.
- It uses both a ssh and a libvirt connection.
- It is desirable in future versions get rid of the ssh connection, but currently
- libvirt does not provide all the needed information.
- Returns (True, Warning) in case of success and (False, <error>) in case of error"""
- warning_text=""
- try:
- #Get virsh and ssh connection
- (return_status, code) = get_ssh_connection(machine, user, password)
- if not return_status:
- print 'RADclass.obtain_RAD() error:', code
- return (return_status, code)
- ssh_conn = code
-
- self.connection_IP = machine
- #print "libvirt open pre"
- virsh_conn=libvirt.open("qemu+ssh://"+user+'@'+machine+"/system")
- #virsh_conn=libvirt.openAuth("qemu+ssh://"+user+'@'+machine+"/system",
- # [[libvirt.VIR_CRED_AUTHNAME, libvirt.VIR_CRED_PASSPHRASE, libvirt.VIR_CRED_USERNAME], getCredentials, password],
- # 0)
- #print "libvirt open after"
-
- # #Set connection infomation
- # (return_status, code) = self.set_connection_info(machine, user, password)
- # if not return_status:
- # return (return_status, 'Error in '+machine+': '+code)
-
- #Set server name
- machine_name = get_hostname(virsh_conn)
- (return_status, code) = self.set_name(machine_name)
- if not return_status:
- return (return_status, 'Error at self.set_name in '+machine+': '+code)
- warning_text += code
-
- #Get the server processors information
- processors = dict()
- (return_status, code) = get_processor_information(ssh_conn, virsh_conn, processors)
- if not return_status:
- return (return_status, 'Error at get_processor_information in '+machine+': '+code)
- warning_text += code
-
- #Get the server memory information
- memory_nodes = dict()
- (return_status, code) = get_memory_information(ssh_conn, virsh_conn, memory_nodes)
- if not return_status:
- return (return_status, 'Error at get_memory_information in '+machine+': '+code)
- warning_text += code
-
- #Get nics information
- nic_topology = dict()
- # (return_status, code) = get_nic_information_old(ssh_conn, nic_topology)
- (return_status, code) = get_nic_information(ssh_conn, virsh_conn, nic_topology)
- if not return_status:
- return (return_status, 'Error at get_nic_information in '+machine+': '+code)
- warning_text += code
-
- #Pack each processor, memory node and nics in a node element
- #and add the node to the RAD element
- for socket_id, processor in processors.iteritems():
- node = Node()
- if not socket_id in nic_topology:
- nic_topology[socket_id] = list()
-
- (return_status, code) = node.set(processor, memory_nodes[socket_id], nic_topology[socket_id])
- # else:
- # (return_status, code) = node.set(processor, memory_nodes[socket_id])
- if not return_status:
- return (return_status, 'Error at node.set in '+machine+': '+code)
- warning_text += code
- (return_status, code) = self.insert_node(node)
- if not return_status:
- return (return_status, 'Error at self.insert_node in '+machine+': '+code)
- if code not in warning_text:
- warning_text += code
-
- #Fill os data
- os = OpSys()
- (return_status, code) = get_os_information(ssh_conn, os)
- if not return_status:
- return (return_status, 'Error at get_os_information in '+machine+': '+code)
- warning_text += code
- (return_status, code) = self.set_os(os)
- if not return_status:
- return (return_status, 'Error at self.set_os in '+machine+': '+code)
- warning_text += code
-
- #Fill hypervisor data
- hypervisor = Hypervisor()
- (return_status, code) = get_hypervisor_information(virsh_conn, hypervisor)
- if not return_status:
- return (return_status, 'Error at get_hypervisor_information in '+machine+': '+code)
- warning_text += code
- (return_status, code) = self.set_hypervisor(hypervisor)
- if not return_status:
- return (return_status, 'Error at self.set_hypervisor in '+machine+': '+code)
- warning_text += code
- ssh_conn.close()
-
- return (True, warning_text)
- except libvirt.libvirtError, e:
- text = e.get_error_message()
- print 'RADclass.obtain_RAD() exception:', text
- return (False, text)
- except paramiko.ssh_exception.SSHException, e:
- text = e.args[0]
- print "obtain_RAD ssh Exception:", text
- return False, text
-
- def set_name(self,name):
- """Sets the machine name.
- Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
- if not isinstance(name,str):
- return (False, 'The variable \'name\' must be text')
- self.name = name
- return (True, "")
-
- def set_connection_info(self, machine, user, password):
- """Sets the connection information.
- Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
- if not isinstance(machine,str):
- return (False, 'The variable \'machine\' must be text')
- if not isinstance(user,str):
- return (False, 'The variable \'user\' must be text')
-# if not isinstance(password,str):
-# return (False, 'The variable \'password\' must be text')
- (self.machine, self.user, self.password) = (machine, user, password)
- return (True, "")
-
- def insert_node(self,node):
- """Inserts a new node and updates class variables.
- Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
- if not isinstance(node,Node):
- return (False, 'The variable \'node\' must be a Node element')
-
- if node.id_ in self.nodes:
- return (False, 'The node is already present in the nodes list.')
-
- #Check if network ports have not been inserted previously as part of another node
- for port_key in node.ports_list:
- if port_key in self.ports_list:
- return (False, 'Network port '+port_key+' defined multiple times in the system')
- self.ports_list.append(port_key)
-
- #Insert the new node
- self.nodes[node.id_] = node
-
- #update variables
- self.update_variables()
-
- return (True, "")
-
- def update_variables(self):
- """Updates class variables.
- Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
- warning_text=""
- #The number of processors and nodes is the same
- self.nr_processors = len(self.nodes)
-
- #If all processors are the same get the values. Otherwise keep them as none
- prev_processor_family = prev_processor_manufacturer = prev_processor_version = prev_processor_features = None
- different_processor_family = different_processor_manufacturer = different_processor_version = different_processor_features = False
- for node in self.nodes.itervalues():
- (self.processor_family, self.processor_manufacturer, self.processor_version, self.processor_features) = node.get_processor_info()
- if prev_processor_family != None and self.processor_family != prev_processor_family:
- different_processor_family = True
- if prev_processor_manufacturer != None and self.processor_manufacturer != prev_processor_manufacturer:
- different_processor_manufacturer = True
- if prev_processor_version != None and self.processor_version != prev_processor_version:
- different_processor_version = True
- if prev_processor_features != None and self.processor_features != prev_processor_features:
- different_processor_features = True
- (prev_processor_family, prev_processor_manufacturer, prev_processor_version, prev_processor_features) = (self.processor_family, self.processor_manufacturer, self.processor_version, self.processor_features)
-
- if different_processor_family:
- self.processor_family = None
- if different_processor_features:
- self.processor_features = None
- if different_processor_manufacturer:
- self.processor_manufacturer = None
- if different_processor_version:
- self.processor_version = None
-
- #If all memory nodes are the same get the values. Otherwise keep them as none
- #Sum the total memory
- self.memory_size = 0
- different_memory_freq = different_memory_nr_channels = different_memory_type = different_memory_hugepage_sz = False
- prev_memory_freq = prev_memory_nr_channels = prev_memory_type = prev_memory_hugepage_sz = None
- for node in self.nodes.itervalues():
- (self.memory_freq, self.memory_nr_channels, self.memory_type, memory_size, self.memory_hugepage_sz) = node.get_memory_info()
- self.memory_size += memory_size
- if prev_memory_freq != None and self.memory_freq != prev_memory_freq:
- different_memory_freq = True
- if prev_memory_nr_channels != None and self.memory_nr_channels != prev_memory_nr_channels:
- different_memory_nr_channels = True
- if prev_memory_type != None and self.memory_type != prev_memory_type:
- different_memory_type = True
- if prev_memory_hugepage_sz != None and self.memory_hugepage_sz != prev_memory_hugepage_sz:
- different_memory_hugepage_sz = True
- (prev_memory_freq, prev_memory_nr_channels, prev_memory_type, prev_memory_hugepage_sz) = (self.memory_freq, self.memory_nr_channels, self.memory_type, self.memory_hugepage_sz)
-
- if different_memory_freq:
- self.memory_freq = None
- if different_memory_nr_channels:
- self.memory_nr_channels = None
- if different_memory_type:
- self.memory_type = None
- if different_memory_hugepage_sz:
- warning_text += 'Detected different hugepages size in different sockets\n'
-
- return (True, warning_text)
-
- def set_hypervisor(self,hypervisor):
- """Sets the hypervisor.
- Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
- if not isinstance(hypervisor,Hypervisor):
- return (False, 'The variable \'hypervisor\' must be of class Hypervisor')
-
- self.hypervisor.assign(hypervisor)
- return (True, "")
-
- def set_os(self,os):
- """Sets the operating system.
- Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
- if not isinstance(os,OpSys):
- return (False, 'The variable \'os\' must be of class OpSys')
-
- self.os.assign(os)
- return (True, "")
-
- def to_text(self):
- text= 'name: '+str(self.name)+'\n'
- text+= 'processor:\n'
- text+= ' nr_processors: '+str(self.nr_processors)+'\n'
- text+= ' family: '+str(self.processor_family)+'\n'
- text+= ' manufacturer: '+str(self.processor_manufacturer)+'\n'
- text+= ' version: '+str(self.processor_version)+'\n'
- text+= ' features: '+str(self.processor_features)+'\n'
- text+= 'memory:\n'
- text+= ' type: '+str(self.memory_type)+'\n'
- text+= ' freq: '+str(self.memory_freq)+'\n'
- text+= ' nr_channels: '+str(self.memory_nr_channels)+'\n'
- text+= ' size: '+str(self.memory_size)+'\n'
- text+= 'hypervisor:\n'
- text+= self.hypervisor.to_text()
- text+= 'os:\n'
- text+= self.os.to_text()
- text+= 'resource topology:\n'
- text+= ' nr_nodes: '+ str(len(self.nodes))+'\n'
- text+= ' nodes:\n'
- for node_k, node_v in self.nodes.iteritems():
- text+= ' node'+str(node_k)+':\n'
- text+= node_v.to_text()
- return text
-
- def to_yaml(self):
- return yaml.load(self.to_text())
-
-class Node():
- def __init__(self):
- self.id_ = None #Integer. Node id. Unique in the system
- self.processor = ProcessorNode() #Information about the processor in the node
- self.memory = MemoryNode() #Information about the memory in the node
- self.nic_list = list() #List of Nic() containing information about the nics associated to the node
- self.ports_list = list() #List containing all network ports in the node. This is used to avoid having defined multiple times the same port in the system
-
- def get_processor_info(self):
- """Gets the processor information. Returns (processor_family, processor_manufacturer, processor_version, processor_features)"""
- return self.processor.get_info()
-
- def get_memory_info(self):
- """Gets the memory information. Returns (memory_freq, memory_nr_channels, memory_type, memory_size)"""
- return self.memory.get_info()
-
-# def set(self, *args):
-# """Sets the node information. Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
-# if len(args)==2:
-# processor = args[0]
-# memory = args[1]
-# nics = False
-# elif len(args)==3:
-# processor = args[0]
-# memory = args[1]
-# nic_list = args[2]
-# nics = True
-# else:
-# return (False, 'Wrong number of elements calling Node().set()')
-
- def set(self, processor, memory, nic_list):
- (status, return_code) = self.processor.assign(processor)
- if not status:
- return (status, return_code)
-
- self.id_ = processor.id_
-
- (status, return_code) = self.memory.assign(memory)
- if not status:
- return (status, return_code)
-
-# if nics:
- for nic in nic_list:
- if not isinstance(nic,Nic):
- return (False, 'The nics must be of type Nic')
- self.nic_list.append(nic)
- for port_key in nic.ports.iterkeys():
- if port_key in self.ports_list:
- return (False, 'Network port '+port_key+'defined multiple times in the same node')
- self.ports_list.append(port_key)
-
- return (True,"")
-
- def assign(self, node):
- """Sets the node information.
- Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
- warning_text=""
- processor = node.processor
- memory = node.memory
- nic_list = node.nic_list
- (status, return_code) = self.processor.assign(processor)
- if not status:
- return (status, return_code)
-
- self.id_ = processor.id_
-
- (status, return_code) = self.memory.assign(memory)
- if not status:
- return (status, return_code)
- warning_text += code
-
- for nic in nic_list:
- if not isinstance(nic,Nic):
- return (False, 'The nics must be of type Nic')
- self.nic_list.append(nic)
- for port_key in nic.ports.iterkeys():
- if port_key in self.ports_list:
- return (False, 'Network port '+port_key+'defined multiple times in the same node')
- self.ports_list.append(port_key)
-
- return (True,warning_text)
-
- def to_text(self):
- text= ' id: '+str(self.id_)+'\n'
- text+= ' cpu:\n'
- text += self.processor.to_text()
- text+= ' memory:\n'
- text += self.memory.to_text()
- if len(self.nic_list) > 0:
- text+= ' nics:\n'
- nic_index = 0
- for nic in self.nic_list:
- text+= ' nic '+str(nic_index)+':\n'
- text += nic.to_text()
- nic_index += 1
- return text
-
-class ProcessorNode():
- #Definition of the possible values of processor variables
- possible_features = definitionsClass.processor_possible_features
- possible_manufacturers = definitionsClass.processor_possible_manufacturers
- possible_families = definitionsClass.processor_possible_families
- possible_versions = definitionsClass.processor_possible_versions
-
- def __init__(self):
- self.id_ = None #Integer. Numeric identifier of the socket
- self.family = None #Text. Family name of the processor
- self.manufacturer = None #Text. Manufacturer of the processor
- self.version = None #Text. Model version of the processor
- self.features = list() #list. List of features offered by the processor
- self.cores = list() #list. List of cores in the processor. In case of hyperthreading the coupled cores are expressed as [a,b]
- self.eligible_cores = list()#list. List of cores that can be used
- #self.decicated_cores
- #self.shared_cores -> this should also contain information to know if cores are being used
-
- def assign(self, processor):
- """Sets the processor information.
- Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
- if not isinstance(processor,ProcessorNode):
- return (False, 'The variable \'processor\' must be of class ProcessorNode')
-
- self.id_ = processor.id_
- self.family = processor.family
- self.manufacturer = processor.manufacturer
- self.version = processor.version
- self.features = processor.features
- self.cores = processor.cores
- self.eligible_cores = processor.eligible_cores
-
- return (True, "")
-
- def set(self, id_, family, manufacturer, version, features, cores):
- """Sets the processor information.
- Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
- warning_text = ""
-
- if not isinstance(id_,int):
- return (False, 'The processor id_ must be of type int')
- if not isinstance(family,str):
- return (False, 'The processor family must be of type str')
- if not isinstance(manufacturer,str):
- return (False, 'The processor manufacturer must be of type str')
- if not isinstance(version,str):
- return (False, 'The processor version must be of type str')
- if not isinstance(features,list):
- return (False, 'The processor features must be of type list')
- if not isinstance(cores,list):
- return (False, 'The processor cores must be of type list')
-
- (self.id_, self.family, self.manufacturer, self.version) = (id_, family, manufacturer, version)
-
- if not manufacturer in self.possible_manufacturers:
- warning_text += "processor manufacturer '%s' not among: %s\n" %(manufacturer, str(self.possible_manufacturers))
- if not family in self.possible_families:
- warning_text += "family '%s' not among: %s\n" % (family, str(self.possible_families))
-# if not version in self.possible_versions:
-# warning_text += 'The version %s is not one of these: %s\n' % (version, str(self.possible_versions))
-
- for feature in features:
- if not feature in self.possible_features:
- warning_text += "processor feature '%s' not among: %s\n" % (feature, str(self.possible_versions))
- self.features.append(feature)
-
- for iterator in sorted(cores):
- if not isinstance(iterator,list) or not all(isinstance(x, int) for x in iterator):
- return (False, 'The cores list must be in the form of [[a,b],[c,d],...] where a,b,c,d are of type int')
- self.cores.append(iterator)
-
- self.set_eligible_cores()
-
- return (True,warning_text)
-
- def set_eligible_cores(self):
- """Set the default eligible cores, this is all cores non used by the host operating system"""
- not_first = False
- for iterator in self.cores:
- if not_first:
- self.eligible_cores.append(iterator)
- else:
- not_first = True
- return
-
- def get_info(self):
- """Returns processor parameters (self.family, self.manufacturer, self.version, self.features)"""
- return (self.family, self.manufacturer, self.version, self.features)
-
- def to_text(self):
- text= ' id: '+str(self.id_)+'\n'
- text+= ' family: '+self.family+'\n'
- text+= ' manufacturer: '+self.manufacturer+'\n'
- text+= ' version: '+self.version+'\n'
- text+= ' features: '+str(self.features)+'\n'
- text+= ' cores: '+str(self.cores)+'\n'
- text+= ' eligible_cores: '+str(self.eligible_cores)+'\n'
- return text
-
-class MemoryNode():
- def __init__(self):
- self.modules = list() #List of MemoryModule(). List of all modules installed in the node
- self.nr_channels = None #Integer. Number of modules installed in the node
- self.node_size = None #Integer. Total size in KiB of memory installed in the node
- self.eligible_memory = None #Integer. Size in KiB of eligible memory in the node
- self.hugepage_sz = None #Integer. Size in KiB of hugepages
- self.hugepage_nr = None #Integer. Number of hugepages allocated in the module
- self.eligible_hugepage_nr = None #Integer. Number of eligible hugepages in the node
- self.type_ = None #Text. Type of memory modules. If modules have a different value keep it as None
- self.freq = None #Integer. Frequency of the modules in MHz. If modules have a different value keep it as None
- self.module_size = None #Integer. Size of the modules in KiB. If modules have a different value keep it as None
- self.form_factor = None #Text. Form factor of the modules. If modules have a different value keep it as None
-
- def assign(self, memory_node):
- return self.set(memory_node.modules, memory_node.hugepage_sz, memory_node.hugepage_nr)
-
- def set(self, modules, hugepage_sz, hugepage_nr):
- """Set the memory node information. hugepage_sz must be expressed in KiB.
- Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
- if not isinstance(modules, list):
- return (False, 'The modules must be a list of elements of class MemoryModule')
- if not isinstance(hugepage_sz,int):
- return (False, 'The hugepage_sz variable must be an int expressing the size in KiB')
- if not isinstance(hugepage_nr,int):
- return (False, 'The hugepage_nr variable must be of type int')
-
- (self.hugepage_sz, self.hugepage_nr) = (hugepage_sz, hugepage_nr)
- self.node_size = self.nr_channels = 0
-
- different_type = different_freq = different_module_size = different_form_factor = False
- prev_type = prev_freq = prev_module_size = prev_form_factor = None
- for iterator in modules:
- if not isinstance(iterator,MemoryModule):
- return (False, 'The modules must be a list of elements of class MemoryModule')
- self.modules.append(iterator)
- (self.type_, self.freq, self.module_size, self.form_factor) = (iterator.type_, iterator.freq, iterator.size, iterator.form_factor)
- self.node_size += self.module_size
- self.nr_channels += 1
- if prev_type != None and prev_type != self.type_:
- different_type = True
- if prev_freq != None and prev_freq != self.freq:
- different_freq = True
- if prev_module_size != None and prev_module_size != self.module_size:
- different_module_size = True
- if prev_form_factor != None and prev_form_factor != self.form_factor:
- different_form_factor = True
- (prev_type, prev_freq, prev_module_size, prev_form_factor) = (self.type_, self.freq, self.module_size, self.form_factor)
-
- if different_type:
- self.type_ = None
- if different_freq:
- self.freq = None
- if different_module_size:
- self.module_size = None
- if different_form_factor:
- self.form_factor = None
-
- (return_value, error_code) = self.set_eligible_memory()
- if not return_value:
- return (return_value, error_code)
-
- return (True, "")
-
- def set_eligible_memory(self):
- """Sets the default eligible_memory and eligible_hugepage_nr. This is all memory but 2GiB and all hugepages"""
- self.eligible_memory = self.node_size - 2*1024*1024
- if self.eligible_memory < 0:
- return (False, "There is less than 2GiB of memory in the module")
-
- self.eligible_hugepage_nr = self.hugepage_nr
- return (True,"")
-
- def get_info(self):
- """Return memory information (self.freq, self.nr_channels, self.type_, self.node_size)"""
- return (self.freq, self.nr_channels, self.type_, self.node_size, self.hugepage_sz)
-
- def to_text(self):
- text= ' node_size: '+str(self.node_size)+'\n'
- text+= ' nr_channels: '+str(self.nr_channels)+'\n'
- text+= ' eligible_memory: '+str(self.eligible_memory)+'\n'
- text+= ' hugepage_sz: '+str(self.hugepage_sz)+'\n'
- text+= ' hugepage_nr: '+str(self.hugepage_nr)+'\n'
- text+= ' eligible_hugepage_nr: '+str(self.eligible_hugepage_nr)+'\n'
- text+= ' type: '+self.type_+'\n'
- text+= ' freq: '+str(self.freq)+'\n'
- text+= ' module_size: '+str(self.module_size)+'\n'
- text+= ' form_factor: '+self.form_factor+'\n'
- text+= ' modules details:\n'
- for module in self.modules:
- text += module.to_text()
- return text
-
-class MemoryModule():
- #Definition of the possible values of module variables
- possible_types = definitionsClass.memory_possible_types
- possible_form_factors = definitionsClass.memory_possible_form_factors
-
- def __init__(self):
- self.locator = None #Text. Name of the memory module
- self.type_ = None #Text. Type of memory module
- self.freq = None #Integer. Frequency of the module in MHz
- self.size = None #Integer. Size of the module in KiB
- self.form_factor = None #Text. Form factor of the module
-
- def set(self, locator, type_, freq, size, form_factor):
- """Sets the memory module information.
- Frequency must be expressed in MHz and size in KiB.
- Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
- warning_text=""
- if not isinstance(locator, str):
- return (False, "The type of the variable locator must be str")
- if not isinstance(type_, str):
- return (False, "The type of the variable type_ must be str")
- if not isinstance(form_factor, str):
- return (False, "The type of the variable form_factor must be str")
- if not isinstance(freq, int):
- return (False, "The type of the variable freq must be int")
- if not isinstance(size, int):
- return (False, "The type of the variable size must be int")
-
- if not form_factor in self.possible_form_factors:
- warning_text += "memory form_factor '%s' not among: %s\n" %(form_factor, str(self.possible_form_factors))
- if not type_ in self.possible_types:
- warning_text += "memory type '%s' not among: %s\n" %(type_, str(self.possible_types))
-
- (self.locator, self.type_, self.freq, self.size, self.form_factor) = (locator, type_, freq, size, form_factor)
- return (True, warning_text)
-
- def to_text(self):
- text= ' '+self.locator+':\n'
- text+= ' type: '+self.type_+'\n'
- text+= ' freq: '+str(self.freq)+'\n'
- text+= ' size: '+str(self.size)+'\n'
- text+= ' form factor: '+self.form_factor+'\n'
- return text
-
-class Nic():
- def __init__(self):
- self.model = None #Text. Model of the nic
- self.ports = dict() #Dictionary of ports. Keys are the port name, value are Port() elements
-
- def set_model(self, model):
- """Sets the model of the nic. Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
- if not isinstance(model,str):
- return (False, 'The \'model\' must be of type str')
-
- self.model = model
- return (True, "")
-
- def add_port(self, port):
- """Adds a port to the nic. Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
- if not isinstance(port,Port):
- return (False, 'The \'port\' must be of class Port')
-
-# port_id = str(port.pci_device_id[0])+':'+str(port.pci_device_id[1])+':'+str(port.pci_device_id[2])+'.'+str(port.pci_device_id[3])
-#CHANGED
-# port_id = port.name
- port_id = port.pci_device_id
-#CHANGED END
- if port_id in self.ports:
- return (False, 'The \'port\' '+port.pci_device_id+' is duplicated in the nic')
-# return (False, 'The \'port\' is duplicated in the nic')
-
- self.ports[port_id] = port
- return (True, "")
-
- def to_text(self):
- text= ' model: '+ str(self.model)+'\n'
- text+= ' ports: '+'\n'
- for key,port in self.ports.iteritems():
- text+= ' "'+key+'":'+'\n'
- text += port.to_text()
- return text
-
-class Port():
- def __init__(self):
- self.name = None #Text. Port name
- self.virtual = None #Boolean. States if the port is a virtual function
- self.enabled = None #Boolean. States if the port is enabled
- self.eligible = None #Boolean. States if the port is eligible
- self.speed = None #Integer. Indicates the speed in Mbps
- self.available_bw = None #Integer. BW in Mbps that is available.
- self.mac = None #list. Indicates the mac address of the port as a list in format ['XX','XX','XX','XX','XX','XX']
- self.pci_device_id_split = None #list. Indicates the pci address of the port as a list in format ['XXXX','XX','XX','X']
- self.pci_device_id = None
- self.PF_pci_device_id = None
-
-# def set(self, name, virtual, enabled, speed, mac, pci_device_id, pci_device_id_split):
-# """Sets the port information. The variable speed indicates the speed in Mbps. Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
-# if not isinstance(name,str):
-# return (False, 'The variable \'name\' must be of type str')
-# if not isinstance(virtual,bool):
-# return (False, 'The variable \'virtual\' must be of type bool')
-# if not isinstance(enabled,bool):
-# return (False, 'The variable \'enabled\' must be of type bool')
-# if not isinstance(enabled,bool):
-# return (speed, 'The variable \'speed\' must be of type int')
-# if not isinstance(mac, list) and not isinstance(mac,NoneType):
-# return (False, 'The variable \'enabled\' must be of type list indicating the mac address in format [\'XXXX\',\'XX\',\'XX\',\'X\'] or NoneType')
-# if not isinstance(pci_device_id_split, list) or len(pci_device_id_split) != 4:
-# return (False, 'The variable \'pci_device_id_split\' must be of type list, indicating the pci address in format [\'XX\',\'XX\',\'XX\',\'XX\',\'XX\',\'XX\']')
-#
-# expected_len = [4,2,2,1]
-# index = 0
-# for iterator in pci_device_id_split:
-# if not isinstance(iterator,str) or not iterator.isdigit() or len(iterator) != expected_len[index]:
-# return (False, 'The variable \'pci_device_id_split\' must be of type list, indicating the pci address in format [\'XX\',\'XX\',\'XX\',\'XX\',\'XX\',\'XX\']')
-# index += 1
-#
-# if not isinstance(mac,NoneType):
-# for iterator in mac:
-# if not isinstance(iterator,str) or not iterator.isalnum() or len(iterator) != 2:
-# return (False, 'The variable \'enabled\' must be of type list indicating the mac address in format [\'XXXX\',\'XX\',\'XX\',\'X\'] or NoneType')
-#
-# #By default only virtual ports are eligible
-# # (self.name, self.virtual, self.enabled, self.eligible, self.available_bw, self.speed, self.mac, self.pci_device_id, self.pci_device_id_split) = (name, virtual, enabled, virtual, speed, speed, mac, pci_device_id, pci_device_id_split)
-# (self.name, self.virtual, self.enabled, self.eligible, self.available_bw, self.mac, self.pci_device_id, self.pci_device_id_split) = (name, virtual, enabled, virtual, speed, mac, pci_device_id, pci_device_id_split)
-
- def to_text(self):
- text= ' pci: "'+ str(self.pci_device_id)+'"\n'
- text+= ' virtual: '+ str(self.virtual)+'\n'
- if self.virtual:
- text+= ' PF_pci_id: "'+self.PF_pci_device_id+'"\n'
- text+= ' eligible: '+ str(self.eligible)+'\n'
- text+= ' enabled: '+str(self.enabled)+'\n'
- text+= ' speed: '+ str(self.speed)+'\n'
- text+= ' available bw: '+ str(self.available_bw)+'\n'
- text+= ' mac: '+ str(self.mac)+'\n'
- text+= ' source_name: '+ str(self.name)+'\n'
- return text
-
-class Hypervisor():
- #Definition of the possible values of hypervisor variables
- possible_types = definitionsClass.hypervisor_possible_types
- possible_domain_types = definitionsClass.hypervisor_possible_domain_types
-
- def __init__(self):
- self.type_ = None #Text. Hypervisor type_
- self.version = None #int. Hypervisor version
- self.lib_version = None #int. Libvirt version used to compile hypervisor
- self.domains = list() #list. List of all the available domains
-
- def set(self, hypervisor, version, lib_version, domains):
- warning_text=""
- if not isinstance(hypervisor,str):
- return (False, 'The variable type_ must be of type str')
- if not isinstance(version,int):
- return (False, 'The variable version must be of type int')
- if not isinstance(lib_version,int):
- return (False, 'The library version must be of type int')
- if not isinstance(domains,list):
- return (False, 'Domains must be a list of the possible domains as str')
-
- if not hypervisor in self.possible_types:
- warning_text += "Hyperpivor '%s' not among: %s\n" % (hypervisor, str(self.possible_types))
-
- valid_domain_found = False
- for domain in domains:
- if not isinstance(domain,str):
- return (False, 'Domains must be a list of the possible domains as str')
- if domain in self.possible_domain_types:
- valid_domain_found = True
- self.domains.append(domain)
-
- if not valid_domain_found:
- warning_text += 'No valid domain found among: %s\n' % str(self.possible_domain_types)
-
-
- (self.version, self.lib_version, self.type_) = (version, lib_version, hypervisor)
- return (True, warning_text)
-
- def assign(self, hypervisor):
- (self.version, self.lib_version, self.type_) = (hypervisor.version, hypervisor.lib_version, hypervisor.type_)
- for domain in hypervisor.domains:
- self.domains.append(domain)
- return
-
- def to_text(self):
- text= ' type: '+self.type_+'\n'
- text+= ' version: '+str(self.version)+'\n'
- text+= ' libvirt version: '+ str(self.lib_version)+'\n'
- text+= ' domains: '+str(self.domains)+'\n'
- return text
-
-class OpSys():
- #Definition of the possible values of os variables
- possible_id = definitionsClass.os_possible_id
- possible_types = definitionsClass.os_possible_types
- possible_architectures = definitionsClass.os_possible_architectures
-
- def __init__(self):
- self.id_ = None #Text. Identifier of the OS. Formed by <Distibutor ID>-<Release>-<Codename>. In linux this can be obtained using lsb_release -a
- self.type_ = None #Text. Type of operating system
- self.bit_architecture = None #Integer. Architecture
-
- def set(self, id_, type_, bit_architecture):
- warning_text=""
- if not isinstance(type_,str):
- return (False, 'The variable type_ must be of type str')
- if not isinstance(id_,str):
- return (False, 'The variable id_ must be of type str')
- if not isinstance(bit_architecture,str):
- return (False, 'The variable bit_architecture must be of type str')
-
- if not type_ in self.possible_types:
- warning_text += "os type '%s' not among: %s\n" %(type_, str(self.possible_types))
- if not id_ in self.possible_id:
- warning_text += "os release '%s' not among: %s\n" %(id_, str(self.possible_id))
- if not bit_architecture in self.possible_architectures:
- warning_text += "os bit_architecture '%s' not among: %s\n" % (bit_architecture, str(self.possible_architectures))
-
- (self.id_, self.type_, self.bit_architecture) = (id_, type_, bit_architecture)
- return (True, warning_text)
-
- def assign(self,os):
- (self.id_, self.type_, self.bit_architecture) = (os.id_, os.type_, os.bit_architecture)
- return
-
- def to_text(self):
- text= ' id: '+self.id_+'\n'
- text+= ' type: '+self.type_+'\n'
- text+= ' bit_architecture: '+self.bit_architecture+'\n'
- return text
-
-def get_hostname(virsh_conn):
- return virsh_conn.getHostname().rstrip('\n')
-
-def get_hugepage_size(ssh_conn):
- command = 'sudo hugeadm --page-sizes'
-# command = 'hugeadm --page-sizes-all'
- (_, stdout, stderr) = ssh_conn.exec_command(command)
- error = stderr.read()
- if len(error)>0:
- raise paramiko.ssh_exception.SSHException(command +' : '+ error)
- mem=stdout.read()
- if mem=="":
- return 0
- return int(mem)
-
-def get_hugepage_nr(ssh_conn,hugepage_sz, node_id):
- command = 'cat /sys/devices/system/node/node'+str(node_id)+'/hugepages/hugepages-'+str(hugepage_sz/1024)+'kB/nr_hugepages'
- (_, stdout, _) = ssh_conn.exec_command(command)
- #print command,
- #text = stdout.read()
- #print "'"+text+"'"
- #return int(text)
-
- try:
- value=int(stdout.read())
- except:
- value=0
- return value
-
-def get_memory_information(ssh_conn, virsh_conn, memory_nodes):
- warning_text=""
- tree=ElementTree.fromstring(virsh_conn.getSysinfo(0))
- memory_dict = dict()
- node_id = 0 #TODO revise. Added for allowing VM as compute hosts
- for target in tree.findall("memory_device"):
- locator_f = size_f = freq_f = type_f = formfactor_f = False
- locator_f = True #TODO revise. Added for allowing VM as compute hosts
- module_form_factor = ""
- for entry in target.findall("entry"):
- if entry.get("name") == 'size':
- size_f = True
- size_split = entry.text.split(' ')
- if size_split[1] == 'MB':
- module_size = int(size_split[0]) * 1024 * 1024
- elif size_split[1] == 'GB':
- module_size = int(size_split[0]) * 1024 * 1024 * 1024
- elif size_split[1] == 'KB':
- module_size = int(size_split[0]) * 1024
- else:
- module_size = int(size_split[0])
-
- elif entry.get("name") == 'speed':
- freq_f = True
- freq_split = entry.text.split(' ')
- if freq_split[1] == 'MHz':
- module_freq = int(freq_split[0]) * 1024 * 1024
- elif freq_split[1] == 'GHz':
- module_freq = int(freq_split[0]) * 1024 * 1024 * 1024
- elif freq_split[1] == 'KHz':
- module_freq = int(freq_split[0]) * 1024
-
- elif entry.get("name") == 'type':
- type_f = True
- module_type = entry.text
-
- elif entry.get("name") == 'form_factor':
- formfactor_f = True
- module_form_factor = entry.text
- #TODO revise. Commented for allowing VM as compute hosts
- # elif entry.get("name") == 'locator' and not locator_f:
- # # other case, it is obtained by bank_locator that we give priority to
- # locator = entry.text
- # pos = locator.find(module_form_factor)
- # if module_form_factor == locator[0:len(module_form_factor) ]:
- # pos = len(module_form_factor) +1
- # else:
- # pos = 0
- # if locator[pos] in "ABCDEFGH":
- # locator_f = True
- # node_id = ord(locator[pos])-ord('A')
- # #print entry.text, node_id
- #
- # elif entry.get("name") == 'bank_locator':
- # locator = entry.text
- # pos = locator.find("NODE ")
- # if pos >= 0 and len(locator)>pos+5:
- # if locator[pos+5] in ("01234567"): #len("NODE ") is 5
- # node_id = int(locator[pos+5])
- # locator_f = True
- #
-
- #When all module fields have been found add a new module to the list
- if locator_f and size_f and freq_f and type_f and formfactor_f:
- #If the memory node has not yet been created create it
- if node_id not in memory_dict:
- memory_dict[node_id] = []
-
- #Add a new module to the memory node
- module = MemoryModule()
- #TODO revise. Changed for allowing VM as compute hosts
- (return_status, code) = module.set('NODE %d' % node_id, module_type, module_freq, module_size, module_form_factor)
- #(return_status, code) = module.set(locator, module_type, module_freq, module_size, module_form_factor)
- if not return_status:
- return (return_status, code)
- memory_dict[node_id].append(module)
- if code not in warning_text:
- warning_text += code
- node_id += 1 #TODO revise. Added for allowing VM as compute hosts
-
- #Fill memory nodes
- #Hugepage size is constant for all nodes
- hugepage_sz = get_hugepage_size(ssh_conn)
- for node_id, modules in memory_dict.iteritems():
- memory_node = MemoryNode()
- memory_node.set(modules, hugepage_sz, get_hugepage_nr(ssh_conn,hugepage_sz, node_id))
- memory_nodes[node_id] = memory_node
-
- return (True, warning_text)
-
-def get_cpu_topology_ht(ssh_conn, topology):
- command = 'cat /proc/cpuinfo'
- (_, stdout, stderr) = ssh_conn.exec_command(command)
- error = stderr.read()
- if len(error)>0:
- raise paramiko.ssh_exception.SSHException(command +' : '+ error)
- sockets = []
- cores = []
- core_map = {}
- core_details = []
- core_lines = {}
- for line in stdout.readlines():
- if len(line.strip()) != 0:
- name, value = line.split(":", 1)
- core_lines[name.strip()] = value.strip()
- else:
- core_details.append(core_lines)
- core_lines = {}
-
- for core in core_details:
- for field in ["processor", "core id", "physical id"]:
- if field not in core:
- return(False,'Error getting '+field+' value from /proc/cpuinfo')
- core[field] = int(core[field])
-
- if core["core id"] not in cores:
- cores.append(core["core id"])
- if core["physical id"] not in sockets:
- sockets.append(core["physical id"])
- key = (core["physical id"], core["core id"])
- if key not in core_map:
- core_map[key] = []
- core_map[key].append(core["processor"])
-
- for s in sockets:
- hyperthreaded_cores = list()
- for c in cores:
- hyperthreaded_cores.append(core_map[(s,c)])
- topology[s] = hyperthreaded_cores
-
- return (True, "")
-
-def get_processor_information(ssh_conn, vish_conn, processors):
- warning_text=""
- #Processor features are the same for all processors
- #TODO (at least using virsh capabilities)nr_numa_nodes
- capabilities = list()
- tree=ElementTree.fromstring(vish_conn.getCapabilities())
- for target in tree.findall("host/cpu/feature"):
- if target.get("name") == 'pdpe1gb':
- capabilities.append('lps')
- elif target.get("name") == 'dca':
- capabilities.append('dioc')
- elif target.get("name") == 'vmx' or target.get("name") == 'svm':
- capabilities.append('hwsv')
- elif target.get("name") == 'ht':
- capabilities.append('ht')
-
- target = tree.find("host/cpu/arch")
- if target.text == 'x86_64' or target.text == 'amd64':
- capabilities.append('64b')
-
- command = 'cat /proc/cpuinfo | grep flags'
- (_, stdout, stderr) = ssh_conn.exec_command(command)
- error = stderr.read()
- if len(error)>0:
- raise paramiko.ssh_exception.SSHException(command +' : '+ error)
- line = stdout.readline()
- if 'ept' in line or 'npt' in line:
- capabilities.append('tlbps')
-
- #Find out if IOMMU is enabled
- command = 'dmesg |grep -e Intel-IOMMU'
- (_, stdout, stderr) = ssh_conn.exec_command(command)
- error = stderr.read()
- if len(error)>0:
- raise paramiko.ssh_exception.SSHException(command +' : '+ error)
- if 'enabled' in stdout.read():
- capabilities.append('iommu')
-
- #Equivalent for AMD
- command = 'dmesg |grep -e AMD-Vi'
- (_, stdout, stderr) = ssh_conn.exec_command(command)
- error = stderr.read()
- if len(error)>0:
- raise paramiko.ssh_exception.SSHException(command +' : '+ error)
- if len(stdout.read()) > 0:
- capabilities.append('iommu')
-
- #-----------------------------------------------------------
- topology = dict()
- #In case hyperthreading is active it is necessary to determine cpu topology using /proc/cpuinfo
- if 'ht' in capabilities:
- (return_status, code) = get_cpu_topology_ht(ssh_conn, topology)
- if not return_status:
- return (return_status, code)
- warning_text += code
-
- #Otherwise it is possible to do it using virsh capabilities
- else:
- for target in tree.findall("host/topology/cells/cell"):
- socket_id = int(target.get("id"))
- topology[socket_id] = list()
- for cpu in target.findall("cpus/cpu"):
- topology[socket_id].append(int(cpu.get("id")))
-
- #-----------------------------------------------------------
- #Create a dictionary with the information of all processors
- #p_fam = p_man = p_ver = None
- tree=ElementTree.fromstring(vish_conn.getSysinfo(0))
- #print vish_conn.getSysinfo(0)
- #return (False, 'forces error for debuging')
- not_populated=False
- socket_id = -1 #in case we can not determine the socket_id we assume incremental order, starting by 0
- for target in tree.findall("processor"):
- count = 0
- socket_id += 1
- #Get processor id, family, manufacturer and version
- for entry in target.findall("entry"):
- if entry.get("name") == "status":
- if entry.text[0:11] == "Unpopulated":
- not_populated=True
- elif entry.get("name") == 'socket_destination':
- socket_text = entry.text
- if socket_text.startswith('CPU'):
- socket_text = socket_text.strip('CPU')
- socket_text = socket_text.strip() #removes trailing spaces
- if socket_text.isdigit() and int(socket_text)<9 and int(socket_text)>0:
- socket_id = int(socket_text) - 1
-
- elif entry.get("name") == 'family':
- family = entry.text
- count += 1
- elif entry.get("name") == 'manufacturer':
- manufacturer = entry.text
- count += 1
- elif entry.get("name") == 'version':
- version = entry.text.strip()
- count += 1
- if count != 3:
- return (False, 'Error. Not all expected fields could be found in processor')
-
- #Create and fill processor structure
- if not_populated:
- continue #avoid inconsistence of some machines where more socket detected than
- processor = ProcessorNode()
- (return_status, code) = processor.set(socket_id, family, manufacturer, version, capabilities, topology[socket_id])
- if not return_status:
- return (return_status, code)
- if code not in warning_text:
- warning_text += code
-
- #Add processor to the processors dictionary
- processors[socket_id] = processor
-
- return (True, warning_text)
-
-def get_nic_information(ssh_conn, virsh_conn, nic_topology):
- warning_text=""
- #Get list of net devices
- net_devices = virsh_conn.listDevices('net',0)
- print virsh_conn.listDevices('net',0)
- for device in net_devices:
- try:
- #Get the XML descriptor of the device:
- net_XML = ElementTree.fromstring(virsh_conn.nodeDeviceLookupByName(device).XMLDesc(0))
- #print "net_XML:" , net_XML
- #obtain the parent
- parent = net_XML.find('parent')
- if parent == None:
- print 'No parent was found in XML for device '+device
- #Error. continue?-------------------------------------------------------------
- continue
- if parent.text == 'computer':
- continue
- if not parent.text.startswith('pci_'):
- print device + ' parent is neither computer nor pci'
- #Error. continue?-------------------------------------------------------------
- continue
- interface = net_XML.find('capability/interface').text
- mac = net_XML.find('capability/address').text
-
- #Get the pci XML
- pci_XML = ElementTree.fromstring(virsh_conn.nodeDeviceLookupByName(parent.text).XMLDesc(0))
- #print pci_XML
- #Get pci
- name = pci_XML.find('name').text.split('_')
- pci = name[1]+':'+name[2]+':'+name[3]+'.'+name[4]
-
- #If slot == 0 it is a PF, otherwise it is a VF
- capability = pci_XML.find('capability')
- if capability.get('type') != 'pci':
- print device + 'Capability is not of type pci in '+parent.text
- #Error. continue?-------------------------------------------------------------
- continue
- slot = capability.find('slot').text
- bus = capability.find('bus').text
- node_id = None
- numa_ = capability.find('numa')
- if numa_ != None:
- node_id = numa_.get('node');
- if node_id != None: node_id =int(node_id)
- if slot == None or bus == None:
- print device + 'Bus and slot not detected in '+parent.text
- #Error. continue?-------------------------------------------------------------
- continue
- if slot != '0':
- # print ElementTree.tostring(pci_XML)
- virtual = True
- capability_pf = capability.find('capability')
- if capability_pf.get('type') != 'phys_function':
- print 'physical_function not found in VF '+parent.text
- #Error. continue?-------------------------------------------------------------
- continue
- PF_pci = capability_pf.find('address').attrib
- PF_pci_text = PF_pci['domain'].split('x')[1]+':'+PF_pci['bus'].split('x')[1]+':'+PF_pci['slot'].split('x')[1]+'.'+PF_pci['function'].split('x')[1]
-
- else:
- virtual = False
-
- #Obtain node for the port
- if node_id == None:
- node_id = int(bus)>>6
- #print "node_id:", node_id
-
- #Only for non virtual interfaces: Obtain speed and if link is detected (this must be done using ethtool)
- if not virtual:
- command = 'sudo ethtool '+interface+' | grep -e Speed -e "Link detected"'
- (_, stdout, stderr) = ssh_conn.exec_command(command)
- error = stderr.read()
- if len(error) >0:
- print 'Error running '+command+'\n'+error
- #Error. continue?-------------------------------------------------------------
- continue
- for line in stdout.readlines():
- line = line.strip().rstrip('\n').split(': ')
- if line[0] == 'Speed':
- if line[1].endswith('Mb/s'):
- speed = int(line[1].split('M')[0])*int(1e6)
- elif line[1].endswith('Gb/s'):
- speed = int(line[1].split('G')[0])*int(1e9)
- elif line[1].endswith('Kb/s'):
- speed = int(line[1].split('K')[0])*int(1e3)
- else:
- #the interface is listed but won't be used
- speed = 0
- elif line[0] == 'Link detected':
- if line[1] == 'yes':
- enabled = True
- else:
- enabled = False
- else:
- print 'Unnexpected output of command '+command+':'
- print line
- #Error. continue?-------------------------------------------------------------
- continue
-
- if not node_id in nic_topology:
- nic_topology[node_id] = list()
- #With this implementation we make the RAD with only one nic per node and this nic has all ports, TODO: change this by including parent information of PF
- nic_topology[node_id].append(Nic())
-
- #Load the appropriate nic
- nic = nic_topology[node_id][0]
-
- #Create a new port and fill it
- port = Port()
- port.name = interface
- port.virtual = virtual
- port.speed = speed
- if virtual:
- port.available_bw = 0
- port.PF_pci_device_id = PF_pci_text
- else:
- port.available_bw = speed
- if speed == 0:
- port.enabled = False
- else:
- port.enabled = enabled
-
- port.eligible = virtual #Only virtual ports are eligible
- port.mac = mac
- port.pci_device_id = pci
- port.pci_device_id_split = name[1:]
-
- #Save the port information
- nic.add_port(port)
- except Exception,e:
- print 'Error: '+str(e)
-
- #set in vitual ports if they are enabled
- for nic in nic_topology.itervalues():
- for port in nic[0].ports.itervalues():
-# print port.pci_device_id
- if port.virtual:
- enabled = nic[0].ports.get(port.PF_pci_device_id)
- if enabled == None:
- return(False, 'The PF '+port.PF_pci_device_id+' (VF '+port.pci_device_id+') is not present in ports dict')
- #Only if the PF is enabled the VF can be enabled
- if nic[0].ports[port.PF_pci_device_id].enabled:
- port.enabled = True
- else:
- port.enabled = False
-
- return (True, warning_text)
-
-def get_nic_information_old(ssh_conn, nic_topology):
- command = 'lstopo-no-graphics --of xml'
- (_, stdout, stderr) = ssh_conn.exec_command(command)
- error = stderr.read()
- if len(error)>0:
- raise paramiko.ssh_exception.SSHException(command +' : '+ error)
- tree=ElementTree.fromstring(stdout.read())
- for target in tree.findall("object/object"):
- #Find numa nodes
- if target.get("type") != "NUMANode":
- continue
- node_id = int(target.get("os_index"))
- nic_topology[node_id] = list()
-
- #find nics in numa node
- for entry in target.findall("object/object"):
- if entry.get("type") != 'Bridge':
- continue
- nic_name = entry.get("name")
- model = None
- nic = Nic()
-
- #find ports in nic
- for pcidev in entry.findall("object"):
- if pcidev.get("type") != 'PCIDev':
- continue
- enabled = speed = mac = pci_busid = None
- port = Port()
- model = pcidev.get("name")
- virtual = False
- if 'Virtual' in model:
- virtual = True
- pci_busid = pcidev.get("pci_busid")
- for osdev in pcidev.findall("object"):
- name = osdev.get("name")
- for info in osdev.findall("info"):
- if info.get("name") != 'Address':
- continue
- mac = info.get("value")
- #get the port speed and status
- command = 'sudo ethtool '+name
- (_, stdout, stderr) = ssh_conn.exec_command(command)
- error = stderr.read()
- if len(error)>0:
- return (False, 'Error obtaining '+name+' information: '+error)
- ethtool = stdout.read()
- if '10000baseT/Full' in ethtool:
- speed = 10e9
- elif '1000baseT/Full' in ethtool:
- speed = 1e9
- elif '100baseT/Full' in ethtool:
- speed = 100e6
- elif '10baseT/Full' in ethtool:
- speed = 10e6
- else:
- return (False, 'Speed not detected in '+name)
-
- enabled = False
- if 'Link detected: yes' in ethtool:
- enabled = True
-
- if speed != None and mac != None and pci_busid != None:
- mac = mac.split(':')
- pci_busid_split = re.split(':|\.', pci_busid)
- #Fill the port information
- port.set(name, virtual, enabled, speed, mac, pci_busid, pci_busid_split)
- nic.add_port(port)
-
- if len(nic.ports) > 0:
- #Fill the nic model
- if model != None:
- nic.set_model(model)
- else:
- nic.set_model(nic_name)
-
- #Add it to the topology
- nic_topology[node_id].append(nic)
-
- return (True, "")
-
-def get_os_information(ssh_conn, os):
- warning_text=""
-# command = 'lsb_release -a'
-# (stdin, stdout, stderr) = ssh_conn.exec_command(command)
-# cont = 0
-# for line in stdout.readlines():
-# line_split = re.split('\t| *', line.rstrip('\n'))
-# if line_split[0] == 'Distributor' and line_split[1] == 'ID:':
-# distributor = line_split[2]
-# cont += 1
-# elif line_split[0] == 'Release:':
-# release = line_split[1]
-# cont += 1
-# elif line_split[0] == 'Codename:':
-# codename = line_split[1]
-# cont += 1
-# if cont != 3:
-# return (False, 'It was not possible to obtain the OS id')
-# id_ = distributor+'-'+release+'-'+codename
-
-
- command = 'cat /etc/redhat-release'
- (_, stdout, _) = ssh_conn.exec_command(command)
- id_text= stdout.read()
- if len(id_text)==0:
- #try with Ubuntu
- command = 'lsb_release -d -s'
- (_, stdout, _) = ssh_conn.exec_command(command)
- id_text= stdout.read()
- if len(id_text)==0:
- raise paramiko.ssh_exception.SSHException("Can not determinte release neither with 'lsb_release' nor with 'cat /etc/redhat-release'")
- id_ = id_text.rstrip('\n')
-
- command = 'uname -o'
- (_, stdout, stderr) = ssh_conn.exec_command(command)
- error = stderr.read()
- if len(error)>0:
- raise paramiko.ssh_exception.SSHException(command +' : '+ error)
- type_ = stdout.read().rstrip('\n')
-
- command = 'uname -i'
- (_, stdout, stderr) = ssh_conn.exec_command(command)
- error = stderr.read()
- if len(error)>0:
- raise paramiko.ssh_exception.SSHException(command +' : '+ error)
- bit_architecture = stdout.read().rstrip('\n')
-
- (return_status, code) = os.set(id_, type_, bit_architecture)
- if not return_status:
- return (return_status, code)
- warning_text += code
- return (True, warning_text)
-
-def get_hypervisor_information(virsh_conn, hypervisor):
- type_ = virsh_conn.getType().rstrip('\n')
- version = virsh_conn.getVersion()
- lib_version = virsh_conn.getLibVersion()
-
- domains = list()
- tree=ElementTree.fromstring(virsh_conn.getCapabilities())
- for target in tree.findall("guest"):
- os_type = target.find("os_type").text
- #We only allow full virtualization
- if os_type != 'hvm':
- continue
- wordsize = int(target.find('arch/wordsize').text)
- if wordsize == 64:
- for domain in target.findall("arch/domain"):
- domains.append(domain.get("type"))
-
- (return_status, code) = hypervisor.set(type_, version, lib_version, domains)
- if not return_status:
- return (return_status, code)
- return (True, code)
-
-class RADavailableResourcesClass(RADclass):
- def __init__(self, resources):
- """Copy resources from the RADclass (server resources not taking into account resources used by VMs"""
- #New
- self.reserved = dict() #Dictionary of reserved resources for a server. Key are VNFC names and values RADreservedResources
- self.cores_consumption = None #Dictionary of cpu consumption. Key is the cpu and the value is
-
- self.machine = resources.machine
- self.user = resources.user
- self.password = resources.password
- self.name = resources.name
- self.nr_processors = resources.nr_processors
- self.processor_family = resources.processor_family
- self.processor_manufacturer = resources.processor_manufacturer
- self.processor_version = resources.processor_version
- self.processor_features = resources.processor_features
- self.memory_type = resources.memory_type
- self.memory_freq = resources.memory_freq
- self.memory_nr_channels = resources.memory_nr_channels
- self.memory_size = resources.memory_size
- self.memory_hugepage_sz = resources.memory_hugepage_sz
- self.hypervisor = Hypervisor()
- self.hypervisor.assign(resources.hypervisor)
- self.os = OpSys()
- self.os.assign(resources.os)
- self.nodes = dict()
- for node_k, node_v in resources.nodes.iteritems():
- self.nodes[node_k] = Node()
- self.nodes[node_k].assign(node_v)
- return
-
- def _get_cores_consumption_warnings(self):
- """Returns list of warning strings in case warnings are generated.
- In case no warnings are generated the return value will be an empty list"""
- warnings = list()
- #Get the cores consumption
- (return_status, code) = get_ssh_connection(self.machine, self.user, self.password)
- if not return_status:
- return (return_status, code)
- ssh_conn = code
- command = 'mpstat -P ALL 1 1 | grep Average | egrep -v CPU\|all'
- (_, stdout, stderr) = ssh_conn.exec_command(command)
- error = stderr.read()
- if len(error) > 0:
- return (False, error)
-
- self.cores_consumption = dict()
- for line in stdout.readlines():
- cpu_usage_split = re.split('\t| *', line.rstrip('\n'))
- usage = 100 *(1 - float(cpu_usage_split[10]))
- if usage > 0:
- self.cores_consumption[int(cpu_usage_split[1])] = usage
- ssh_conn.close()
- #Check if any core marked as available in the nodes has cpu_usage > 0
- for _, node_v in self.nodes.iteritems():
- cores = node_v.processor.eligible_cores
- for cpu in cores:
- if len(cpu) > 1:
- for core in cpu:
- if core in self.cores_consumption:
- warnings.append('Warning: Core '+str(core)+' is supposed to be idle but it is consuming '+str(self.cores_consumption[core])+'%')
- else:
- if cpu in self.cores_consumption:
- warnings.append('Warning: Core '+str(core)+' is supposed to be idle but it is consuming '+str(self.cores_consumption[cpu])+'%')
-
- return warnings
-
- def reserved_to_text(self):
- text = str()
- for VNFC_name, VNFC_reserved in self.reserved.iteritems():
- text += ' VNFC: '+str(VNFC_name)+'\n'
- text += VNFC_reserved.to_text()
-
- return text
-
- def obtain_usage(self):
- resp = dict()
- #Iterate through nodes to get cores, eligible cores, memory and physical ports (save ports usage for next section)
- nodes = dict()
- ports_usage = dict()
- hugepage_size = dict()
- for node_k, node_v in self.nodes.iteritems():
- node = dict()
- ports_usage[node_k] = dict()
- eligible_cores = list()
- for pair in node_v.processor.eligible_cores:
- if isinstance(pair, list):
- for element in pair:
- eligible_cores.append(element)
- else:
- eligible_cores.append(pair)
- node['cpus'] = {'cores':node_v.processor.cores,'eligible_cores':eligible_cores}
- node['memory'] = {'size':str(node_v.memory.node_size/(1024*1024*1024))+'GB','eligible':str(node_v.memory.eligible_memory/(1024*1024*1024))+'GB'}
- hugepage_size[node_k] = node_v.memory.hugepage_sz
-
- ports = dict()
- for nic in node_v.nic_list:
- for port in nic.ports.itervalues():
- if port.enabled and not port.virtual:
- ports[port.name] = {'speed':str(port.speed/1000000000)+'G'}
-# print '*************** ',port.name,'speed',port.speed
- ports_usage[node_k][port.name] = 100 - int(100*float(port.available_bw)/float(port.speed))
- node['ports'] = ports
- nodes[node_k] = node
- resp['RAD'] = nodes
-
- #Iterate through reserved section to get used cores, used memory and port usage
- cores = dict()
- memory = dict()
- #reserved_cores = list
- for node_k in self.nodes.iterkeys():
- if not node_k in cores:
- cores[node_k] = list()
- memory[node_k] = 0
- for _, reserved in self.reserved.iteritems():
- if node_k in reserved.node_reserved_resources:
- node_v = reserved.node_reserved_resources[node_k]
- cores[node_k].extend(node_v.reserved_cores)
- memory[node_k] += node_v.reserved_hugepage_nr * hugepage_size[node_k]
-
- occupation = dict()
- for node_k in self.nodes.iterkeys():
- ports = dict()
- for name, usage in ports_usage[node_k].iteritems():
- ports[name] = {'occupied':str(usage)+'%'}
-# print '****************cores',cores
-# print '****************memory',memory
- occupation[node_k] = {'cores':cores[node_k],'memory':str(memory[node_k]/(1024*1024*1024))+'GB','ports':ports}
- resp['occupation'] = occupation
-
- return resp
-
-class RADreservedResources():
- def __init__(self):
- self.node_reserved_resources = dict() #dict. keys are the RAD nodes id, values are NodeReservedResources
- self.mgmt_interface_pci = None #pci in the VNF for the management interface
- self.image = None #Path in remote machine of the VNFC image
-
- def update(self,reserved):
- self.image = reserved.image
- self.mgmt_interface_pci = reserved.mgmt_interface_pci
- for k,v in reserved.node_reserved_resources.iteritems():
- if k in self.node_reserved_resources.keys():
- return (False, 'Duplicated node entry '+str(k)+' in reserved resources')
- self.node_reserved_resources[k]=v
-
- return (True, "")
-
- def to_text(self):
- text = ' image: '+str(self.image)+'\n'
- for node_id, node_reserved in self.node_reserved_resources.iteritems():
- text += ' Node ID: '+str(node_id)+'\n'
- text += node_reserved.to_text()
- return text
-
-class NodeReservedResources():
- def __init__(self):
- # reserved_shared_cores = None #list. List of all cores that the VNFC needs in shared mode #TODO Not used
- # reserved_memory = None #Integer. Amount of KiB needed by the VNFC #TODO. Not used since hugepages are used
- self.reserved_cores = list() #list. List of all cores that the VNFC uses
- self.reserved_hugepage_nr = 0 #Integer. Number of hugepages needed by the VNFC
- self.reserved_ports = dict() #dict. The key is the physical port pci and the value the VNFC port description
- self.vlan_tags = dict()
- self.cpu_pinning = None
-
- def to_text(self):
- text = ' cores: '+str(self.reserved_cores)+'\n'
- text += ' cpu_pinning: '+str(self.cpu_pinning)+'\n'
- text += ' hugepages_nr: '+str(self.reserved_hugepage_nr)+'\n'
- for port_pci, port_description in self.reserved_ports.iteritems():
- text += ' port: '+str(port_pci)+'\n'
- text += port_description.to_text()
- return text
-
-# def update(self,reserved):
-# self.reserved_cores = list(reserved.reserved_cores)
-# self.reserved_hugepage_nr = reserved.reserved_hugepage_nr
-# self.reserved_ports = dict(reserved.reserved_ports)
-# self.cpu_pinning = list(reserved.cpu_pinning)
-
-
-
projects / osm / openvim.git / blobdiff