osm_openvim/RADclass.py

# -*- coding: utf-8 -*-

##
# Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U.
# This file is part of openvim
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# For those usages not covered by the Apache License, Version 2.0 please
# contact with: nfvlabs@tid.es
##

'''
Implement the logic for obtaining compute nodes information 
Resource Availability Descriptor 
'''
__author__="Pablo Montes"

#TODO: remove warnings, remove unused things 

from definitionsClass import definitionsClass
from auxiliary_functions import get_ssh_connection
import libvirt
from xml.etree import ElementTree
import paramiko 
import re
import yaml


def getCredentials(creds, data):
    """Used as a backup for libvirt.openAuth in order to provide password that came with data,
    not used by the moment
    """
    print "RADclass:getCredentials", creds, data
    for cred in creds:
        print cred[1] + ": ",
        if cred[0] == libvirt.VIR_CRED_AUTHNAME:
            cred[4] = data
        elif cred[0] == libvirt.VIR_CRED_PASSPHRASE:
            cred[4] = data
        else:
            return -1
    return 0

class RADclass():
    def __init__(self):
        self.name = None
        self.machine = None
        self.user = None
        self.password = None
        self.nodes = dict()                 #Dictionary of nodes. Keys are the node id, values are Node() elements
        self.nr_processors = None           #Integer. Number of processors in the system 
        self.processor_family = None        #If all nodes have the same value equal them, otherwise keep as None
        self.processor_manufacturer = None  #If all nodes have the same value equal them, otherwise keep as None
        self.processor_version = None       #If all nodes have the same value equal them, otherwise keep as None
        self.processor_features = None      #If all nodes have the same value equal them, otherwise keep as None
        self.memory_type = None             #If all nodes have the same value equal them, otherwise keep as None
        self.memory_freq = None             #If all nodes have the same value equal them, otherwise keep as None
        self.memory_nr_channels = None      #If all nodes have the same value equal them, otherwise keep as None
        self.memory_size = None             #Integer. Sum of the memory in all nodes
        self.memory_hugepage_sz = None
        self.hypervisor = Hypervisor()      #Hypervisor information
        self.os = OpSys()                   #Operating system information
        self.ports_list = list()            #List containing all network ports in the node. This is used to avoid having defined multiple times the same port in the system
    
    
    def obtain_RAD(self, user, password, machine):
        """This function obtains the RAD information from the remote server.
        It uses both a ssh and a libvirt connection. 
        It is desirable in future versions get rid of the ssh connection, but currently 
        libvirt does not provide all the needed information. 
        Returns (True, Warning) in case of success and (False, <error>) in case of error"""
        warning_text=""
        try:
            #Get virsh and ssh connection
            (return_status, code) = get_ssh_connection(machine, user, password)
            if not return_status:
                print 'RADclass.obtain_RAD() error:', code
                return (return_status, code)
            ssh_conn = code
            
            self.connection_IP = machine
            #print "libvirt open pre"
            virsh_conn=libvirt.open("qemu+ssh://"+user+'@'+machine+"/system")
            #virsh_conn=libvirt.openAuth("qemu+ssh://"+user+'@'+machine+"/system", 
            #        [[libvirt.VIR_CRED_AUTHNAME, libvirt.VIR_CRED_PASSPHRASE, libvirt.VIR_CRED_USERNAME], getCredentials, password],
            #        0)
            #print "libvirt open after"
            
    #         #Set connection infomation
    #         (return_status, code) = self.set_connection_info(machine, user, password)
    #         if not return_status:
    #             return (return_status, 'Error in '+machine+': '+code)
            
            #Set server name
            machine_name = get_hostname(virsh_conn)
            (return_status, code) = self.set_name(machine_name)
            if not return_status:
                return (return_status, 'Error at self.set_name in '+machine+': '+code)
            warning_text += code
            
            #Get the server processors information
            processors = dict()
            (return_status, code) = get_processor_information(ssh_conn, virsh_conn, processors)
            if not return_status:
                return (return_status, 'Error at get_processor_information in '+machine+': '+code)
            warning_text += code
            
            #Get the server memory information
            memory_nodes = dict()
            (return_status, code) = get_memory_information(ssh_conn, virsh_conn, memory_nodes)
            if not return_status:
                return (return_status, 'Error at get_memory_information in '+machine+': '+code)
            warning_text += code
            
            #Get nics information
            nic_topology = dict()
    #         (return_status, code) = get_nic_information_old(ssh_conn, nic_topology)
            (return_status, code) = get_nic_information(ssh_conn, virsh_conn, nic_topology)
            if not return_status:
                return (return_status, 'Error at get_nic_information in '+machine+': '+code)
            warning_text += code
            
            #Pack each processor, memory node  and nics in a node element
            #and add the node to the RAD element
            for socket_id, processor in processors.iteritems():
                node = Node()
                if not socket_id in nic_topology:
                    nic_topology[socket_id] = list()
                    
                (return_status, code) = node.set(processor, memory_nodes[socket_id], nic_topology[socket_id])
    #             else:
    #                 (return_status, code) = node.set(processor, memory_nodes[socket_id])
                if not return_status:
                    return (return_status, 'Error at node.set in '+machine+': '+code)
                warning_text += code
                (return_status, code) = self.insert_node(node)
                if not return_status:
                    return (return_status, 'Error at self.insert_node in '+machine+': '+code)
                if code not in warning_text:
                    warning_text += code
            
            #Fill os data
            os = OpSys()
            (return_status, code) = get_os_information(ssh_conn, os)
            if not return_status:
                return (return_status, 'Error at get_os_information in '+machine+': '+code)
            warning_text += code
            (return_status, code) = self.set_os(os)
            if not return_status:
                return (return_status, 'Error at self.set_os in '+machine+': '+code)
            warning_text += code
            
            #Fill hypervisor data
            hypervisor = Hypervisor()
            (return_status, code) = get_hypervisor_information(virsh_conn, hypervisor)
            if not return_status:
                return (return_status, 'Error at get_hypervisor_information in '+machine+': '+code)
            warning_text += code
            (return_status, code) = self.set_hypervisor(hypervisor)
            if not return_status:
                return (return_status, 'Error at self.set_hypervisor in '+machine+': '+code)
            warning_text += code
            ssh_conn.close()
                
            return (True, warning_text)
        except libvirt.libvirtError, e:
            text = e.get_error_message()
            print 'RADclass.obtain_RAD() exception:', text
            return (False, text)
        except paramiko.ssh_exception.SSHException, e:
            text = e.args[0]
            print  "obtain_RAD ssh Exception:", text
            return False, text

    def set_name(self,name):
        """Sets the machine name. 
        Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
        if not isinstance(name,str):
            return (False, 'The variable \'name\' must be text')
        self.name = name
        return (True, "")
    
    def set_connection_info(self, machine, user, password):
        """Sets the connection information. 
        Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
        if not isinstance(machine,str):
            return (False, 'The variable \'machine\' must be text')
        if not isinstance(user,str):
            return (False, 'The variable \'user\' must be text')
#         if not isinstance(password,str):
#             return (False, 'The variable \'password\' must be text')
        (self.machine, self.user, self.password) = (machine, user, password)
        return (True, "")
        
    def insert_node(self,node):
        """Inserts a new node and updates class variables. 
        Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
        if not isinstance(node,Node):
            return (False, 'The variable \'node\' must be a Node element')
        
        if node.id_ in self.nodes:
            return (False, 'The node is already present in the nodes list.')
        
        #Check if network ports have not been inserted previously as part of another node
        for port_key in node.ports_list:
            if port_key in self.ports_list:
                return (False, 'Network port '+port_key+' defined multiple times in the system')
            self.ports_list.append(port_key)
        
        #Insert the new node
        self.nodes[node.id_] = node
        
        #update variables
        self.update_variables()
        
        return (True, "")
    
    def update_variables(self):
        """Updates class variables. 
        Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
        warning_text=""
        #The number of processors and nodes is the same
        self.nr_processors = len(self.nodes)
        
        #If all processors are the same get the values. Otherwise keep them as none
        prev_processor_family = prev_processor_manufacturer = prev_processor_version = prev_processor_features = None
        different_processor_family = different_processor_manufacturer = different_processor_version = different_processor_features = False
        for node in self.nodes.itervalues():
            (self.processor_family, self.processor_manufacturer, self.processor_version, self.processor_features) = node.get_processor_info()
            if prev_processor_family != None and self.processor_family != prev_processor_family:
                different_processor_family = True
            if prev_processor_manufacturer != None and self.processor_manufacturer != prev_processor_manufacturer:
                different_processor_manufacturer = True
            if prev_processor_version != None and self.processor_version != prev_processor_version:
                different_processor_version = True
            if prev_processor_features != None and self.processor_features != prev_processor_features:
                different_processor_features = True
            (prev_processor_family, prev_processor_manufacturer, prev_processor_version, prev_processor_features) = (self.processor_family, self.processor_manufacturer, self.processor_version, self.processor_features)

        if different_processor_family:
            self.processor_family = None
        if different_processor_features:
            self.processor_features = None
        if different_processor_manufacturer:
            self.processor_manufacturer = None
        if different_processor_version:
            self.processor_version = None
            
        #If all memory nodes are the same get the values. Otherwise keep them as none
        #Sum the total memory
        self.memory_size = 0
        different_memory_freq = different_memory_nr_channels = different_memory_type = different_memory_hugepage_sz = False
        prev_memory_freq = prev_memory_nr_channels = prev_memory_type = prev_memory_hugepage_sz = None
        for node in self.nodes.itervalues():
            (self.memory_freq, self.memory_nr_channels, self.memory_type, memory_size, self.memory_hugepage_sz) = node.get_memory_info()
            self.memory_size += memory_size 
            if prev_memory_freq != None and self.memory_freq != prev_memory_freq:
                different_memory_freq = True
            if prev_memory_nr_channels != None and self.memory_nr_channels != prev_memory_nr_channels:
                different_memory_nr_channels = True
            if prev_memory_type != None and self.memory_type != prev_memory_type:
                different_memory_type = True
            if prev_memory_hugepage_sz != None and self.memory_hugepage_sz != prev_memory_hugepage_sz:
                different_memory_hugepage_sz = True
            (prev_memory_freq, prev_memory_nr_channels, prev_memory_type, prev_memory_hugepage_sz) = (self.memory_freq, self.memory_nr_channels, self.memory_type, self.memory_hugepage_sz)
            
        if different_memory_freq:
            self.memory_freq = None
        if different_memory_nr_channels:
            self.memory_nr_channels = None
        if different_memory_type:
            self.memory_type = None
        if different_memory_hugepage_sz:
            warning_text += 'Detected different hugepages size in different sockets\n'
            
        return (True, warning_text)
        
    def set_hypervisor(self,hypervisor):
        """Sets the hypervisor. 
        Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
        if not isinstance(hypervisor,Hypervisor):
            return (False, 'The variable \'hypervisor\' must be of class Hypervisor')
        
        self.hypervisor.assign(hypervisor) 
        return (True, "")
    
    def set_os(self,os):
        """Sets the operating system. 
        Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
        if not isinstance(os,OpSys):
            return (False, 'The variable \'os\' must be of class OpSys')
        
        self.os.assign(os)
        return (True, "")
    
    def to_text(self):
        text= 'name: '+str(self.name)+'\n'
        text+= 'processor:\n'
        text+= '    nr_processors: '+str(self.nr_processors)+'\n' 
        text+= '    family: '+str(self.processor_family)+'\n'
        text+= '    manufacturer: '+str(self.processor_manufacturer)+'\n'
        text+= '    version: '+str(self.processor_version)+'\n'
        text+= '    features: '+str(self.processor_features)+'\n'
        text+= 'memory:\n'
        text+= '    type: '+str(self.memory_type)+'\n'
        text+= '    freq: '+str(self.memory_freq)+'\n'
        text+= '    nr_channels: '+str(self.memory_nr_channels)+'\n'
        text+= '    size: '+str(self.memory_size)+'\n'
        text+= 'hypervisor:\n'
        text+= self.hypervisor.to_text()
        text+= 'os:\n'
        text+= self.os.to_text()
        text+= 'resource topology:\n'
        text+= '    nr_nodes: '+ str(len(self.nodes))+'\n'
        text+= '    nodes:\n'
        for node_k, node_v in self.nodes.iteritems():
            text+= '        node'+str(node_k)+':\n'
            text+= node_v.to_text()
        return text
    
    def to_yaml(self):
        return yaml.load(self.to_text())
    
class Node():
    def __init__(self):
        self.id_ = None                      #Integer. Node id. Unique in the system
        self.processor = ProcessorNode()    #Information about the processor in the node
        self.memory = MemoryNode()          #Information about the memory in the node
        self.nic_list = list()              #List of Nic() containing information about the nics associated to the node
        self.ports_list = list()            #List containing all network ports in the node. This is used to avoid having defined multiple times the same port in the system
        
    def get_processor_info(self):
        """Gets the processor information. Returns (processor_family, processor_manufacturer, processor_version, processor_features)"""
        return self.processor.get_info()
        
    def get_memory_info(self):
        """Gets the memory information. Returns (memory_freq, memory_nr_channels, memory_type, memory_size)"""
        return self.memory.get_info()
    
#     def set(self, *args):
#         """Sets the node information. Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
#         if len(args)==2:
#             processor = args[0]
#             memory = args[1]
#             nics = False
#         elif len(args)==3:
#             processor = args[0]
#             memory = args[1]
#             nic_list = args[2]
#             nics = True
#         else:
#             return (False, 'Wrong number of elements calling Node().set()')

    def set(self, processor, memory, nic_list):
        (status, return_code) = self.processor.assign(processor)
        if not status:
            return (status, return_code)
        
        self.id_ = processor.id_
        
        (status, return_code) = self.memory.assign(memory)
        if not status:
            return (status, return_code)
        
#         if nics:
        for nic in nic_list:
            if not isinstance(nic,Nic):
                return (False, 'The nics must be of type Nic')
            self.nic_list.append(nic)
            for port_key in nic.ports.iterkeys():
                if port_key in self.ports_list:
                    return (False, 'Network port '+port_key+'defined multiple times in the same node')
                self.ports_list.append(port_key)
            
        return (True,"")
   
    def assign(self, node):
        """Sets the node information. 
        Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
        warning_text=""
        processor = node.processor
        memory = node.memory
        nic_list = node.nic_list
        (status, return_code) = self.processor.assign(processor)
        if not status:
            return (status, return_code)
        
        self.id_ = processor.id_
        
        (status, return_code) = self.memory.assign(memory)
        if not status:
            return (status, return_code)
        warning_text += code
        
        for nic in nic_list:
            if not isinstance(nic,Nic):
                return (False, 'The nics must be of type Nic')
            self.nic_list.append(nic)
            for port_key in nic.ports.iterkeys():
                if port_key in self.ports_list:
                    return (False, 'Network port '+port_key+'defined multiple times in the same node')
                self.ports_list.append(port_key)
            
        return (True,warning_text)
   
    def to_text(self):
        text= '            id: '+str(self.id_)+'\n'
        text+= '            cpu:\n'
        text += self.processor.to_text()
        text+= '            memory:\n'
        text += self.memory.to_text()
        if len(self.nic_list) > 0:
            text+= '            nics:\n'
            nic_index = 0
            for nic in self.nic_list:
                text+= '                nic '+str(nic_index)+':\n'
                text += nic.to_text()
                nic_index += 1
        return text
    
class ProcessorNode():
    #Definition of the possible values of processor variables
    possible_features = definitionsClass.processor_possible_features
    possible_manufacturers = definitionsClass.processor_possible_manufacturers
    possible_families = definitionsClass.processor_possible_families
    possible_versions = definitionsClass.processor_possible_versions
    
    def __init__(self):
        self.id_ = None              #Integer. Numeric identifier of the socket
        self.family = None          #Text. Family name of the processor
        self.manufacturer = None    #Text. Manufacturer of the processor
        self.version = None         #Text. Model version of the processor
        self.features = list()      #list. List of features offered by the processor
        self.cores = list()         #list. List of cores in the processor. In case of hyperthreading the coupled cores are expressed as [a,b]
        self.eligible_cores = list()#list. List of cores that can be used
        #self.decicated_cores
        #self.shared_cores -> this should also contain information to know if cores are being used
        
    def assign(self, processor):
        """Sets the processor information. 
        Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
        if not isinstance(processor,ProcessorNode):
            return (False, 'The variable \'processor\' must be of class ProcessorNode')
        
        self.id_ = processor.id_
        self.family = processor.family
        self.manufacturer = processor.manufacturer
        self.version = processor.version
        self.features = processor.features
        self.cores = processor.cores
        self.eligible_cores = processor.eligible_cores
        
        return (True, "")
    
    def set(self, id_, family, manufacturer, version, features, cores):
        """Sets the processor information. 
        Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
        warning_text = ""

        if not isinstance(id_,int):
            return (False, 'The processor id_ must be of type int')  
        if not isinstance(family,str):
            return (False, 'The processor family must be of type str')
        if not isinstance(manufacturer,str):
            return (False, 'The processor manufacturer must be of type str')
        if not isinstance(version,str):
            return (False, 'The processor version must be of type str')        
        if not isinstance(features,list):
            return (False, 'The processor features must be of type list')
        if not isinstance(cores,list):
            return (False, 'The processor cores must be of type list')
 
        (self.id_, self.family, self.manufacturer, self.version) = (id_, family, manufacturer, version)
 
        if not manufacturer in self.possible_manufacturers:
            warning_text += "processor manufacturer '%s' not among: %s\n" %(manufacturer, str(self.possible_manufacturers))     
        if not family in self.possible_families:
            warning_text += "family '%s' not among: %s\n" % (family, str(self.possible_families))
#        if not version in self.possible_versions:
#            warning_text += 'The version %s is not one of these: %s\n' % (version, str(self.possible_versions))
        
        for feature in features:
            if not feature in self.possible_features:
                warning_text += "processor feature '%s' not among: %s\n" % (feature, str(self.possible_versions))
            self.features.append(feature)
        
        for iterator in sorted(cores):
            if not isinstance(iterator,list) or not all(isinstance(x, int) for x in iterator):
                return (False, 'The cores list must be in the form of [[a,b],[c,d],...] where a,b,c,d are of type int')
            self.cores.append(iterator)
        
        self.set_eligible_cores()
        
        return (True,warning_text)
           
    def set_eligible_cores(self):
        """Set the default eligible cores, this is all cores non used by the host operating system"""
        not_first = False
        for iterator in self.cores:
            if not_first:
                self.eligible_cores.append(iterator)
            else:
                not_first = True                
        return
    
    def get_info(self):
        """Returns processor parameters (self.family, self.manufacturer, self.version, self.features)"""
        return (self.family, self.manufacturer, self.version, self.features)
    
    def to_text(self):
        text= '                id: '+str(self.id_)+'\n'
        text+= '                family: '+self.family+'\n'
        text+= '                manufacturer: '+self.manufacturer+'\n'
        text+= '                version: '+self.version+'\n'
        text+= '                features: '+str(self.features)+'\n'
        text+= '                cores: '+str(self.cores)+'\n'
        text+= '                eligible_cores: '+str(self.eligible_cores)+'\n'
        return text
    
class MemoryNode():
    def __init__(self):
        self.modules = list()               #List of MemoryModule(). List of all modules installed in the node
        self.nr_channels = None             #Integer. Number of modules installed in the node
        self.node_size = None               #Integer. Total size in KiB of memory installed in the node
        self.eligible_memory = None         #Integer. Size in KiB of eligible memory in the node     
        self.hugepage_sz = None             #Integer. Size in KiB of hugepages
        self.hugepage_nr = None             #Integer. Number of hugepages allocated in the module
        self.eligible_hugepage_nr = None    #Integer. Number of eligible hugepages in the node
        self.type_ = None                    #Text. Type of memory modules. If modules have a different value keep it as None
        self.freq = None                    #Integer. Frequency of the modules in MHz. If modules have a different value keep it as None
        self.module_size = None             #Integer. Size of the modules in KiB. If modules have a different value keep it as None
        self.form_factor = None             #Text. Form factor of the modules. If modules have a different value keep it as None
       
    def assign(self, memory_node):
        return self.set(memory_node.modules, memory_node.hugepage_sz, memory_node.hugepage_nr)
         
    def set(self, modules, hugepage_sz, hugepage_nr):
        """Set the memory node information. hugepage_sz must be expressed in KiB. 
        Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
        if not isinstance(modules, list):
            return (False, 'The modules must be a list of elements of class MemoryModule')
        if not isinstance(hugepage_sz,int):
            return (False, 'The hugepage_sz variable must be an int expressing the size in KiB')
        if not isinstance(hugepage_nr,int):
            return (False, 'The hugepage_nr variable must be of type int')
        
        (self.hugepage_sz, self.hugepage_nr) = (hugepage_sz, hugepage_nr)
        self.node_size = self.nr_channels = 0
        
        different_type = different_freq = different_module_size = different_form_factor = False
        prev_type = prev_freq = prev_module_size = prev_form_factor = None
        for iterator in modules:
            if not isinstance(iterator,MemoryModule):
                return (False, 'The modules must be a list of elements of class MemoryModule')
            self.modules.append(iterator)
            (self.type_, self.freq, self.module_size, self.form_factor) = (iterator.type_, iterator.freq, iterator.size, iterator.form_factor)
            self.node_size += self.module_size
            self.nr_channels += 1
            if prev_type != None and prev_type != self.type_:
                different_type = True
            if prev_freq != None and prev_freq != self.freq:
                different_freq = True
            if prev_module_size != None and prev_module_size != self.module_size:
                different_module_size = True
            if prev_form_factor != None and prev_form_factor != self.form_factor:
                different_form_factor = True
            (prev_type, prev_freq, prev_module_size, prev_form_factor) = (self.type_, self.freq, self.module_size, self.form_factor)
        
        if different_type:
            self.type_ = None
        if different_freq:
            self.freq = None
        if different_module_size:
            self.module_size = None
        if different_form_factor:
            self.form_factor = None
        
        (return_value, error_code) = self.set_eligible_memory()
        if not return_value:
            return (return_value, error_code)
        
        return (True, "")
    
    def set_eligible_memory(self):
        """Sets the default eligible_memory and eligible_hugepage_nr. This is all memory but 2GiB and all hugepages"""
        self.eligible_memory = self.node_size - 2*1024*1024
        if self.eligible_memory < 0:
            return (False, "There is less than 2GiB of memory in the module")
        
        self.eligible_hugepage_nr = self.hugepage_nr 
        return (True,"")
    
    def get_info(self):
        """Return memory information (self.freq, self.nr_channels, self.type_, self.node_size)"""
        return (self.freq, self.nr_channels, self.type_, self.node_size, self.hugepage_sz)
        
    def to_text(self):
        text= '                node_size: '+str(self.node_size)+'\n'
        text+= '                nr_channels: '+str(self.nr_channels)+'\n'
        text+= '                eligible_memory: '+str(self.eligible_memory)+'\n'
        text+= '                hugepage_sz: '+str(self.hugepage_sz)+'\n'
        text+= '                hugepage_nr: '+str(self.hugepage_nr)+'\n'
        text+= '                eligible_hugepage_nr: '+str(self.eligible_hugepage_nr)+'\n'
        text+= '                type: '+self.type_+'\n'
        text+= '                freq: '+str(self.freq)+'\n'
        text+= '                module_size: '+str(self.module_size)+'\n'
        text+= '                form_factor: '+self.form_factor+'\n'
        text+= '                modules details:\n'
        for module in self.modules:
            text += module.to_text()
        return text
        
class MemoryModule():
    #Definition of the possible values of module variables
    possible_types = definitionsClass.memory_possible_types
    possible_form_factors = definitionsClass.memory_possible_form_factors
    
    def __init__(self):
        self.locator = None     #Text. Name of the memory module
        self.type_ = None        #Text. Type of memory module
        self.freq = None        #Integer. Frequency of the module in MHz
        self.size = None        #Integer. Size of the module in KiB
        self.form_factor = None #Text. Form factor of the module
        
    def set(self, locator, type_, freq, size, form_factor):
        """Sets the memory module information. 
        Frequency must be expressed in MHz and size in KiB.
        Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
        warning_text=""
        if not isinstance(locator, str):
            return (False, "The type of the variable locator must be str")
        if not isinstance(type_, str):
            return (False, "The type of the variable type_ must be str")
        if not isinstance(form_factor, str):
            return (False, "The type of the variable form_factor must be str")
        if not isinstance(freq, int):
            return (False, "The type of the variable freq must be int")
        if not isinstance(size, int):
            return (False, "The type of the variable size must be int")
        
        if not form_factor in self.possible_form_factors:
            warning_text += "memory form_factor '%s' not among: %s\n" %(form_factor, str(self.possible_form_factors))
        if not type_ in self.possible_types:
            warning_text += "memory type '%s' not among: %s\n" %(type_, str(self.possible_types))
        
        (self.locator, self.type_, self.freq, self.size, self.form_factor) = (locator, type_, freq, size, form_factor)
        return (True, warning_text)   
    
    def to_text(self):
        text= '                    '+self.locator+':\n'
        text+= '                        type: '+self.type_+'\n'
        text+= '                        freq: '+str(self.freq)+'\n'
        text+= '                        size: '+str(self.size)+'\n'
        text+= '                        form factor: '+self.form_factor+'\n'
        return text
         
class Nic():
    def __init__(self):
        self.model = None       #Text. Model of the nic
        self.ports = dict()     #Dictionary of ports. Keys are the port name, value are Port() elements
    
    def set_model(self, model):
        """Sets the model of the nic. Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
        if not isinstance(model,str):
            return (False, 'The \'model\' must be of type str')
           
        self.model = model
        return (True, "")
   
    def add_port(self, port):
        """Adds a port to the nic. Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
        if not isinstance(port,Port):
            return (False, 'The \'port\' must be of class Port')
       
#        port_id = str(port.pci_device_id[0])+':'+str(port.pci_device_id[1])+':'+str(port.pci_device_id[2])+'.'+str(port.pci_device_id[3])
#CHANGED
#        port_id = port.name
        port_id = port.pci_device_id
#CHANGED END 
        if port_id in self.ports:
            return (False, 'The \'port\' '+port.pci_device_id+' is duplicated in the nic')
#             return (False, 'The \'port\' is duplicated in the nic')
       
        self.ports[port_id] = port
        return (True, "")
   
    def to_text(self):
        text= '                    model: '+ str(self.model)+'\n'
        text+= '                    ports: '+'\n'
        for key,port in self.ports.iteritems():
            text+= '                        "'+key+'":'+'\n'
            text += port.to_text()
        return text
               
class Port():
    def __init__(self):
        self.name = None            #Text. Port name
        self.virtual = None         #Boolean. States if the port is a virtual function
        self.enabled = None         #Boolean. States if the port is enabled
        self.eligible = None        #Boolean. States if the port is eligible
        self.speed = None           #Integer. Indicates the speed in Mbps
        self.available_bw = None    #Integer. BW in Mbps that is available.
        self.mac = None             #list. Indicates the mac address of the port as a list in format ['XX','XX','XX','XX','XX','XX']
        self.pci_device_id_split = None   #list. Indicates the pci address  of the port as a list in format ['XXXX','XX','XX','X']
        self.pci_device_id = None
        self.PF_pci_device_id = None
        
#     def set(self, name, virtual, enabled, speed, mac, pci_device_id, pci_device_id_split):
#         """Sets the port information. The variable speed indicates the speed in Mbps. Returns (True,Warning) in case of success and ('False',<error description>) in case of error"""
#         if not isinstance(name,str):
#             return (False, 'The variable \'name\' must be of type str')
#         if not isinstance(virtual,bool):
#             return (False, 'The variable \'virtual\' must be of type bool')
#         if not isinstance(enabled,bool):
#             return (False, 'The variable \'enabled\' must be of type bool')
#         if not isinstance(enabled,bool):
#             return (speed, 'The variable \'speed\' must be of type int')
#         if not isinstance(mac, list) and not isinstance(mac,NoneType):
#             return (False, 'The variable \'enabled\' must be of type list indicating the mac address in format [\'XXXX\',\'XX\',\'XX\',\'X\'] or NoneType')
#         if not isinstance(pci_device_id_split, list) or len(pci_device_id_split) != 4: 
#             return (False, 'The variable \'pci_device_id_split\' must be of type list, indicating the pci address in format [\'XX\',\'XX\',\'XX\',\'XX\',\'XX\',\'XX\']')
#         
#         expected_len = [4,2,2,1]
#         index = 0
#         for iterator in pci_device_id_split:
#             if not isinstance(iterator,str) or not iterator.isdigit() or len(iterator) != expected_len[index]:
#                 return (False, 'The variable \'pci_device_id_split\' must be of type list, indicating the pci address in format [\'XX\',\'XX\',\'XX\',\'XX\',\'XX\',\'XX\']')
#             index += 1
#             
#         if not isinstance(mac,NoneType):
#             for iterator in mac:
#                 if not isinstance(iterator,str) or not iterator.isalnum() or len(iterator) != 2:
#                     return (False, 'The variable \'enabled\' must be of type list indicating the mac address in format [\'XXXX\',\'XX\',\'XX\',\'X\'] or NoneType')
#         
#         #By default only virtual ports are eligible
# #         (self.name, self.virtual, self.enabled, self.eligible, self.available_bw, self.speed, self.mac, self.pci_device_id, self.pci_device_id_split) = (name, virtual, enabled, virtual, speed, speed, mac, pci_device_id, pci_device_id_split)
#         (self.name, self.virtual, self.enabled, self.eligible, self.available_bw, self.mac, self.pci_device_id, self.pci_device_id_split) = (name, virtual, enabled, virtual, speed, mac, pci_device_id, pci_device_id_split)

    def to_text(self):
        text= '                            pci: "'+ str(self.pci_device_id)+'"\n'
        text+= '                            virtual: '+ str(self.virtual)+'\n'
        if self.virtual:
            text+= '                            PF_pci_id: "'+self.PF_pci_device_id+'"\n'
        text+= '                            eligible: '+ str(self.eligible)+'\n'
        text+= '                            enabled: '+str(self.enabled)+'\n'
        text+= '                            speed: '+ str(self.speed)+'\n'
        text+= '                            available bw: '+ str(self.available_bw)+'\n'
        text+= '                            mac: '+ str(self.mac)+'\n'
        text+= '                            source_name: '+ str(self.name)+'\n'
        return text
    
class Hypervisor():
    #Definition of the possible values of hypervisor variables
    possible_types = definitionsClass.hypervisor_possible_types
    possible_domain_types = definitionsClass.hypervisor_possible_domain_types

    def __init__(self):
        self.type_ = None            #Text. Hypervisor type_
        self.version = None         #int. Hypervisor version
        self.lib_version = None     #int. Libvirt version used to compile hypervisor
        self.domains = list()       #list. List of all the available domains
        
    def set(self, hypervisor, version, lib_version, domains):
        warning_text=""
        if not isinstance(hypervisor,str):
            return (False, 'The variable type_ must be of type str')
        if not isinstance(version,int):
            return (False, 'The variable version must be of type int')
        if not isinstance(lib_version,int):
            return (False, 'The library version must be of type int')
        if not isinstance(domains,list):
            return (False, 'Domains must be a list of the possible domains as str')
        
        if not hypervisor in self.possible_types:
            warning_text += "Hyperpivor '%s' not among: %s\n" % (hypervisor, str(self.possible_types))
        
        valid_domain_found = False
        for domain in domains:
            if not isinstance(domain,str):
                return (False, 'Domains must be a list of the possible domains as str')
            if domain in self.possible_domain_types:
                valid_domain_found = True
                self.domains.append(domain)
                
        if not valid_domain_found:
            warning_text += 'No valid domain found among: %s\n' % str(self.possible_domain_types)
            
        
        (self.version, self.lib_version, self.type_) = (version, lib_version, hypervisor)
        return (True, warning_text)
     
    def assign(self, hypervisor):
        (self.version, self.lib_version, self.type_) = (hypervisor.version, hypervisor.lib_version, hypervisor.type_)
        for domain in hypervisor.domains:
            self.domains.append(domain)
        return
           
    def to_text(self):
        text= '    type: '+self.type_+'\n'
        text+= '    version: '+str(self.version)+'\n'
        text+= '    libvirt version: '+ str(self.lib_version)+'\n'
        text+= '    domains: '+str(self.domains)+'\n'
        return text
        
class OpSys():
    #Definition of the possible values of os variables
    possible_id = definitionsClass.os_possible_id
    possible_types = definitionsClass.os_possible_types
    possible_architectures = definitionsClass.os_possible_architectures

    def __init__(self):
        self.id_ = None                   #Text. Identifier of the OS. Formed by <Distibutor ID>-<Release>-<Codename>. In linux this can be obtained using lsb_release -a
        self.type_ = None                 #Text. Type of operating system
        self.bit_architecture = None     #Integer. Architecture
        
    def set(self, id_, type_, bit_architecture):
        warning_text=""
        if not isinstance(type_,str):
            return (False, 'The variable type_ must be of type str')
        if not isinstance(id_,str):
            return (False, 'The variable id_ must be of type str')
        if not isinstance(bit_architecture,str):
            return (False, 'The variable bit_architecture must be of type str')
        
        if not type_ in self.possible_types:
            warning_text += "os type '%s' not among: %s\n" %(type_, str(self.possible_types))
        if not id_ in self.possible_id:
            warning_text += "os release '%s' not among: %s\n" %(id_, str(self.possible_id))
        if not bit_architecture in self.possible_architectures:
            warning_text += "os bit_architecture '%s' not among: %s\n" % (bit_architecture, str(self.possible_architectures))
        
        (self.id_, self.type_, self.bit_architecture) = (id_, type_, bit_architecture)
        return (True, warning_text)
    
    def assign(self,os):
        (self.id_, self.type_, self.bit_architecture) = (os.id_, os.type_, os.bit_architecture)
        return
    
    def to_text(self):
        text= '    id: '+self.id_+'\n'
        text+= '    type: '+self.type_+'\n'
        text+= '    bit_architecture: '+self.bit_architecture+'\n'
        return text
     
def get_hostname(virsh_conn):
    return virsh_conn.getHostname().rstrip('\n')

def get_hugepage_size(ssh_conn):
    command = 'sudo hugeadm --page-sizes'
#  command = 'hugeadm --page-sizes-all'
    (_, stdout, stderr) = ssh_conn.exec_command(command)
    error = stderr.read()
    if len(error)>0:
        raise paramiko.ssh_exception.SSHException(command +' : '+ error)
    mem=stdout.read()
    if mem=="":
        return 0
    return int(mem)

def get_hugepage_nr(ssh_conn,hugepage_sz, node_id):
    command = 'cat /sys/devices/system/node/node'+str(node_id)+'/hugepages/hugepages-'+str(hugepage_sz/1024)+'kB/nr_hugepages'
    (_, stdout, _) = ssh_conn.exec_command(command)
    #print command, 
    #text = stdout.read()
    #print "'"+text+"'"
    #return int(text)
    
    try:
        value=int(stdout.read())
    except: 
        value=0
    return value

def get_memory_information(ssh_conn, virsh_conn, memory_nodes):
    warning_text=""
    tree=ElementTree.fromstring(virsh_conn.getSysinfo(0))
    memory_dict = dict()
    node_id = 0 #TODO revise. Added for allowing VM as compute hosts 
    for target in tree.findall("memory_device"):
        locator_f = size_f = freq_f = type_f = formfactor_f = False
        locator_f = True #TODO revise. Added for allowing VM as compute hosts
        module_form_factor = ""
        for entry in target.findall("entry"):
            if entry.get("name") == 'size':
                size_f = True
                size_split = entry.text.split(' ')
                if size_split[1] == 'MB':
                    module_size = int(size_split[0]) * 1024 * 1024
                elif size_split[1] == 'GB':
                    module_size = int(size_split[0]) * 1024 * 1024 * 1024
                elif size_split[1] == 'KB':
                    module_size = int(size_split[0]) * 1024
                else:
                    module_size = int(size_split[0])
                
            elif entry.get("name") == 'speed':
                freq_f = True
                freq_split = entry.text.split(' ')
                if freq_split[1] == 'MHz':
                    module_freq = int(freq_split[0]) * 1024 * 1024
                elif freq_split[1] == 'GHz':
                    module_freq = int(freq_split[0]) * 1024 * 1024 * 1024
                elif freq_split[1] == 'KHz':
                    module_freq = int(freq_split[0]) * 1024
            
            elif entry.get("name") == 'type':
                type_f = True
                module_type = entry.text
                   
            elif entry.get("name") == 'form_factor':
                formfactor_f = True
                module_form_factor = entry.text  
            #TODO revise. Commented for allowing VM as compute hosts
            # elif entry.get("name") == 'locator' and not locator_f:
            #     # other case, it is obtained by bank_locator that we give priority to
            #     locator = entry.text
            #     pos = locator.find(module_form_factor)
            #     if module_form_factor == locator[0:len(module_form_factor) ]:
            #         pos = len(module_form_factor) +1 
            #     else:
            #         pos = 0
            #     if locator[pos] in "ABCDEFGH":  
            #         locator_f = True
            #         node_id = ord(locator[pos])-ord('A')
            #         #print entry.text, node_id
            # 
            # elif entry.get("name") == 'bank_locator':
            #     locator = entry.text
            #     pos = locator.find("NODE ")
            #     if pos >= 0 and len(locator)>pos+5:
            #         if locator[pos+5] in ("01234567"): #len("NODE ") is 5
            #             node_id = int(locator[pos+5])
            #             locator_f = True
            #  

        #When all module fields have been found add a new module to the list 
        if locator_f and size_f and freq_f and type_f and formfactor_f:
            #If the memory node has not yet been created create it
            if node_id not in memory_dict:
                memory_dict[node_id] = []
                
            #Add a new module to the memory node
            module = MemoryModule()
            #TODO revise. Changed for allowing VM as compute hosts
            (return_status, code) = module.set('NODE %d' % node_id, module_type, module_freq, module_size, module_form_factor)
            #(return_status, code) = module.set(locator, module_type, module_freq, module_size, module_form_factor)
            if not return_status:
                return (return_status, code)
            memory_dict[node_id].append(module)
            if code not in warning_text:
                warning_text += code
            node_id += 1 #TODO revise. Added for allowing VM as compute hosts
    
    #Fill memory nodes
    #Hugepage size is constant for all nodes
    hugepage_sz = get_hugepage_size(ssh_conn)
    for node_id, modules in memory_dict.iteritems():
        memory_node = MemoryNode()
        memory_node.set(modules, hugepage_sz, get_hugepage_nr(ssh_conn,hugepage_sz, node_id))
        memory_nodes[node_id] = memory_node
        
    return (True, warning_text)

def get_cpu_topology_ht(ssh_conn, topology):
    command = 'cat /proc/cpuinfo'
    (_, stdout, stderr) = ssh_conn.exec_command(command)
    error = stderr.read()
    if len(error)>0:
        raise paramiko.ssh_exception.SSHException(command +' : '+ error)
    sockets = []
    cores = []
    core_map = {}
    core_details = []
    core_lines = {}
    for line in stdout.readlines():
        if len(line.strip()) != 0:
            name, value = line.split(":", 1)
            core_lines[name.strip()] = value.strip()
        else:
            core_details.append(core_lines)
            core_lines = {}
    
    for core in core_details:
        for field in ["processor", "core id", "physical id"]:
            if field not in core:
                return(False,'Error getting '+field+' value from /proc/cpuinfo')
            core[field] = int(core[field])
    
        if core["core id"] not in cores:
            cores.append(core["core id"])
        if core["physical id"] not in sockets:
            sockets.append(core["physical id"])
        key = (core["physical id"], core["core id"])
        if key not in core_map:
            core_map[key] = []
        core_map[key].append(core["processor"])
      
    for s in sockets:
        hyperthreaded_cores = list()
        for c in cores:
            hyperthreaded_cores.append(core_map[(s,c)])
        topology[s] = hyperthreaded_cores
      
    return (True, "")

def get_processor_information(ssh_conn, vish_conn, processors):
    warning_text=""
    #Processor features are the same for all processors
    #TODO (at least using virsh capabilities)nr_numa_nodes
    capabilities = list()
    tree=ElementTree.fromstring(vish_conn.getCapabilities())
    for target in tree.findall("host/cpu/feature"):
        if target.get("name") == 'pdpe1gb':
            capabilities.append('lps')
        elif target.get("name") == 'dca':
            capabilities.append('dioc')  
        elif target.get("name") == 'vmx' or target.get("name") == 'svm':
            capabilities.append('hwsv')
        elif target.get("name") == 'ht':
            capabilities.append('ht')
        
    target = tree.find("host/cpu/arch")
    if target.text == 'x86_64' or target.text == 'amd64':
        capabilities.append('64b')
      
    command = 'cat /proc/cpuinfo | grep flags'
    (_, stdout, stderr) = ssh_conn.exec_command(command)
    error = stderr.read()
    if len(error)>0:
        raise paramiko.ssh_exception.SSHException(command +' : '+ error)
    line = stdout.readline()
    if 'ept' in line or 'npt' in line:
        capabilities.append('tlbps')
    
    #Find out if IOMMU is enabled
    command = 'dmesg |grep -e Intel-IOMMU'
    (_, stdout, stderr) = ssh_conn.exec_command(command)
    error = stderr.read()
    if len(error)>0:
        raise paramiko.ssh_exception.SSHException(command +' : '+ error)
    if 'enabled' in stdout.read():
        capabilities.append('iommu')
      
    #Equivalent for AMD
    command = 'dmesg |grep -e AMD-Vi'
    (_, stdout, stderr) = ssh_conn.exec_command(command)
    error = stderr.read()
    if len(error)>0:
        raise paramiko.ssh_exception.SSHException(command +' : '+ error)
    if len(stdout.read()) > 0:
        capabilities.append('iommu')
    
    #-----------------------------------------------------------
    topology = dict()
    #In case hyperthreading is active it is necessary to determine cpu topology using /proc/cpuinfo
    if 'ht' in capabilities:
        (return_status, code) = get_cpu_topology_ht(ssh_conn, topology)
        if not return_status:
            return (return_status, code)
        warning_text += code

    #Otherwise it is possible to do it using virsh capabilities
    else:
        for target in tree.findall("host/topology/cells/cell"):
            socket_id = int(target.get("id"))
            topology[socket_id] = list()
            for cpu in target.findall("cpus/cpu"):
                topology[socket_id].append(int(cpu.get("id")))
    
    #-----------------------------------------------------------         
    #Create a dictionary with the information of all processors
    #p_fam = p_man = p_ver = None
    tree=ElementTree.fromstring(vish_conn.getSysinfo(0))
    #print vish_conn.getSysinfo(0)
    #return (False, 'forces error for debuging')
    not_populated=False
    socket_id = -1     #in case we can not determine the socket_id we assume incremental order, starting by 0
    for target in tree.findall("processor"):
        count = 0
        socket_id += 1
        #Get processor id, family, manufacturer and version
        for entry in target.findall("entry"):
            if entry.get("name") == "status":
                if entry.text[0:11] == "Unpopulated":
                    not_populated=True
            elif entry.get("name") == 'socket_destination':
                socket_text = entry.text
                if socket_text.startswith('CPU'):
                    socket_text = socket_text.strip('CPU')
                    socket_text = socket_text.strip() #removes trailing spaces
                    if socket_text.isdigit() and int(socket_text)<9 and int(socket_text)>0:
                        socket_id = int(socket_text) - 1
              
            elif entry.get("name") == 'family':
                family = entry.text
                count += 1
            elif entry.get("name") == 'manufacturer':
                manufacturer = entry.text
                count += 1
            elif entry.get("name") == 'version':
                version = entry.text.strip()
                count += 1
        if count != 3:
            return (False, 'Error. Not all expected fields could be found in processor')
        
        #Create and fill processor structure
        if not_populated:
            continue  #avoid inconsistence of some machines where more socket detected than 
        processor = ProcessorNode()
        (return_status, code) = processor.set(socket_id, family, manufacturer, version, capabilities, topology[socket_id])
        if not return_status:
            return (return_status, code)
        if code not in warning_text:
            warning_text += code

        #Add processor to the processors dictionary
        processors[socket_id] = processor
    
    return (True, warning_text)

def get_nic_information(ssh_conn, virsh_conn, nic_topology):   
    warning_text=""
    #Get list of net devices
    net_devices = virsh_conn.listDevices('net',0)
    print virsh_conn.listDevices('net',0)
    for device in net_devices:
        try:
            #Get the XML descriptor of the device:
            net_XML = ElementTree.fromstring(virsh_conn.nodeDeviceLookupByName(device).XMLDesc(0))
            #print "net_XML:" , net_XML
            #obtain the parent
            parent = net_XML.find('parent')
            if parent == None:
                print 'No parent was found in XML for device '+device
                #Error. continue?-------------------------------------------------------------
                continue
            if parent.text == 'computer':
                continue
            if not parent.text.startswith('pci_'):
                print device + ' parent is neither computer nor pci'
                #Error. continue?-------------------------------------------------------------
                continue
            interface = net_XML.find('capability/interface').text
            mac = net_XML.find('capability/address').text
            
            #Get the pci XML
            pci_XML = ElementTree.fromstring(virsh_conn.nodeDeviceLookupByName(parent.text).XMLDesc(0))
            #print pci_XML
            #Get pci
            name = pci_XML.find('name').text.split('_')
            pci = name[1]+':'+name[2]+':'+name[3]+'.'+name[4]
            
            #If slot == 0 it is a PF, otherwise it is a VF
            capability = pci_XML.find('capability')
            if capability.get('type') != 'pci':
                print device + 'Capability is not of type pci in '+parent.text
                #Error. continue?-------------------------------------------------------------
                continue
            slot = capability.find('slot').text
            bus = capability.find('bus').text
            node_id = None
            numa_ = capability.find('numa')
            if numa_ != None:
                node_id = numa_.get('node');
                if node_id != None: node_id =int(node_id)
            if slot == None or bus == None:
                print device + 'Bus and slot not detected in '+parent.text
                #Error. continue?-------------------------------------------------------------
                continue
            if slot != '0':
    #             print ElementTree.tostring(pci_XML)
                virtual = True
                capability_pf = capability.find('capability')
                if capability_pf.get('type') != 'phys_function':
                    print 'physical_function not found in VF '+parent.text
                    #Error. continue?-------------------------------------------------------------
                    continue
                PF_pci = capability_pf.find('address').attrib
                PF_pci_text = PF_pci['domain'].split('x')[1]+':'+PF_pci['bus'].split('x')[1]+':'+PF_pci['slot'].split('x')[1]+'.'+PF_pci['function'].split('x')[1]
                
            else:
                virtual = False
            
            #Obtain node for the port
            if node_id == None:
                node_id = int(bus)>>6
            #print "node_id:", node_id
            
            #Only for non virtual interfaces: Obtain speed and if link is detected (this must be done using ethtool)
            if not virtual:
                command = 'sudo ethtool '+interface+' | grep -e Speed -e "Link detected"'
                (_, stdout, stderr) = ssh_conn.exec_command(command)
                error = stderr.read()
                if len(error) >0:
                    print 'Error running '+command+'\n'+error
                    #Error. continue?-------------------------------------------------------------
                    continue
                for line in stdout.readlines():
                    line = line.strip().rstrip('\n').split(': ')
                    if line[0] == 'Speed':
                        if line[1].endswith('Mb/s'):
                            speed = int(line[1].split('M')[0])*int(1e6)
                        elif line[1].endswith('Gb/s'):
                            speed = int(line[1].split('G')[0])*int(1e9)
                        elif line[1].endswith('Kb/s'):
                            speed = int(line[1].split('K')[0])*int(1e3)
                        else:
                            #the interface is listed but won't be used
                            speed = 0
                    elif line[0] == 'Link detected':
                        if line[1] == 'yes':
                            enabled = True
                        else:
                            enabled = False
                    else:
                        print 'Unnexpected output of command '+command+':'
                        print line
                        #Error. continue?-------------------------------------------------------------
                        continue
                
            if not node_id in nic_topology:
                nic_topology[node_id] = list()
                #With this implementation we make the RAD with only one nic per node and this nic has all ports, TODO: change this by including parent information of PF
                nic_topology[node_id].append(Nic())
             
            #Load the appropriate nic    
            nic = nic_topology[node_id][0]
            
            #Create a new port and fill it
            port = Port()
            port.name = interface
            port.virtual = virtual
            port.speed = speed
            if virtual:
                port.available_bw = 0
                port.PF_pci_device_id = PF_pci_text
            else:
                port.available_bw = speed
                if speed == 0:
                    port.enabled = False
                else:
                    port.enabled = enabled

            port.eligible = virtual  #Only virtual ports are eligible
            port.mac = mac
            port.pci_device_id = pci
            port.pci_device_id_split = name[1:]
            
            #Save the port information
            nic.add_port(port)         
        except Exception,e:
            print 'Error: '+str(e)

    #set in vitual ports if they are enabled
    for nic in nic_topology.itervalues():
        for port in nic[0].ports.itervalues():
#             print port.pci_device_id
            if port.virtual:
                enabled = nic[0].ports.get(port.PF_pci_device_id)
                if enabled == None:
                    return(False, 'The PF '+port.PF_pci_device_id+' (VF '+port.pci_device_id+') is not present in ports dict')
                #Only if the PF is enabled the VF can be enabled
                if nic[0].ports[port.PF_pci_device_id].enabled:
                    port.enabled = True
                else:
                    port.enabled = False
            
    return (True, warning_text)     

def get_nic_information_old(ssh_conn, nic_topology):
    command = 'lstopo-no-graphics --of xml'
    (_, stdout, stderr) = ssh_conn.exec_command(command)
    error = stderr.read()
    if len(error)>0:
        raise paramiko.ssh_exception.SSHException(command +' : '+ error)
    tree=ElementTree.fromstring(stdout.read())
    for target in tree.findall("object/object"):
        #Find numa nodes
        if target.get("type") != "NUMANode":
            continue
        node_id = int(target.get("os_index"))
        nic_topology[node_id] = list()
        
        #find nics in numa node
        for entry in target.findall("object/object"):
            if entry.get("type") != 'Bridge':
                continue
            nic_name = entry.get("name")
            model = None
            nic = Nic()
            
            #find ports in nic
            for pcidev in entry.findall("object"):
                if pcidev.get("type") != 'PCIDev':
                    continue
                enabled = speed = mac = pci_busid = None
                port = Port()
                model = pcidev.get("name")
                virtual = False
                if 'Virtual' in model:
                    virtual = True
                pci_busid = pcidev.get("pci_busid")
                for osdev in pcidev.findall("object"):
                    name = osdev.get("name")
                    for info in osdev.findall("info"):
                        if info.get("name") != 'Address':
                            continue
                        mac = info.get("value")
                        #get the port speed and status
                        command = 'sudo ethtool '+name
                        (_, stdout, stderr) = ssh_conn.exec_command(command)
                        error = stderr.read()
                        if len(error)>0:
                            return (False, 'Error obtaining '+name+' information: '+error)
                        ethtool = stdout.read()
                        if '10000baseT/Full' in ethtool:
                            speed = 10e9
                        elif '1000baseT/Full' in ethtool:
                            speed = 1e9
                        elif '100baseT/Full' in ethtool:
                            speed = 100e6
                        elif '10baseT/Full' in ethtool:
                            speed = 10e6
                        else:
                            return (False, 'Speed not detected in '+name)

                    enabled = False
                    if 'Link detected: yes' in ethtool:
                        enabled = True
                    
                    if speed != None and mac != None and pci_busid != None:
                        mac = mac.split(':')
                        pci_busid_split = re.split(':|\.', pci_busid)
                        #Fill the port information
                        port.set(name, virtual, enabled, speed, mac, pci_busid, pci_busid_split)
                        nic.add_port(port)
              
            if len(nic.ports) > 0:  
                #Fill the nic model
                if model != None:
                    nic.set_model(model)
                else:
                    nic.set_model(nic_name)
                
                #Add it to the topology
                nic_topology[node_id].append(nic)
                
    return (True, "")

def get_os_information(ssh_conn, os):
    warning_text=""
#    command = 'lsb_release -a'
#    (stdin, stdout, stderr) = ssh_conn.exec_command(command)
#    cont = 0
#    for line in stdout.readlines():
#        line_split = re.split('\t| *', line.rstrip('\n'))
#        if line_split[0] == 'Distributor' and line_split[1] == 'ID:':
#            distributor = line_split[2]
#            cont += 1
#        elif line_split[0] == 'Release:':
#            release = line_split[1]
#            cont += 1
#        elif line_split[0] == 'Codename:':
#            codename = line_split[1]
#            cont += 1
#    if cont != 3:
#        return (False, 'It was not possible to obtain the OS id')
#    id_ = distributor+'-'+release+'-'+codename


    command = 'cat /etc/redhat-release'
    (_, stdout, _) = ssh_conn.exec_command(command)
    id_text= stdout.read()
    if len(id_text)==0:
        #try with Ubuntu
        command = 'lsb_release -d -s'
        (_, stdout, _) = ssh_conn.exec_command(command)
        id_text= stdout.read()
    if len(id_text)==0:
        raise paramiko.ssh_exception.SSHException("Can not determinte release neither with 'lsb_release' nor with 'cat /etc/redhat-release'")
    id_ = id_text.rstrip('\n')
   
    command = 'uname -o'
    (_, stdout, stderr) = ssh_conn.exec_command(command)
    error = stderr.read()
    if len(error)>0:
        raise paramiko.ssh_exception.SSHException(command +' : '+ error)
    type_ = stdout.read().rstrip('\n')
    
    command = 'uname -i'
    (_, stdout, stderr) = ssh_conn.exec_command(command)
    error = stderr.read()
    if len(error)>0:
        raise paramiko.ssh_exception.SSHException(command +' : '+ error)
    bit_architecture = stdout.read().rstrip('\n')
    
    (return_status, code) = os.set(id_, type_, bit_architecture)
    if not return_status:
        return (return_status, code)
    warning_text += code
    return (True, warning_text) 

def get_hypervisor_information(virsh_conn, hypervisor):
    type_ = virsh_conn.getType().rstrip('\n')
    version = virsh_conn.getVersion()
    lib_version = virsh_conn.getLibVersion()
    
    domains = list()
    tree=ElementTree.fromstring(virsh_conn.getCapabilities())
    for target in tree.findall("guest"):
        os_type = target.find("os_type").text
        #We only allow full virtualization
        if os_type != 'hvm':
            continue
        wordsize = int(target.find('arch/wordsize').text)
        if wordsize == 64:
            for domain in target.findall("arch/domain"):
                domains.append(domain.get("type"))
            
    (return_status, code) = hypervisor.set(type_, version, lib_version, domains)
    if not return_status:
        return (return_status, code)
    return (True, code)      
     
class RADavailableResourcesClass(RADclass):
    def __init__(self, resources):
        """Copy resources from the RADclass (server resources not taking into account resources used by VMs"""
        #New
        self.reserved = dict()          #Dictionary of reserved resources for a server. Key are VNFC names and values RADreservedResources
        self.cores_consumption = None   #Dictionary of cpu consumption. Key is the cpu and the value is
        
        self.machine = resources.machine
        self.user = resources.user
        self.password = resources.password
        self.name = resources.name
        self.nr_processors = resources.nr_processors 
        self.processor_family = resources.processor_family
        self.processor_manufacturer = resources.processor_manufacturer
        self.processor_version = resources.processor_version
        self.processor_features = resources.processor_features
        self.memory_type = resources.memory_type
        self.memory_freq = resources.memory_freq
        self.memory_nr_channels = resources.memory_nr_channels
        self.memory_size = resources.memory_size
        self.memory_hugepage_sz = resources.memory_hugepage_sz
        self.hypervisor = Hypervisor()
        self.hypervisor.assign(resources.hypervisor)
        self.os = OpSys()
        self.os.assign(resources.os)
        self.nodes = dict()
        for node_k, node_v in resources.nodes.iteritems():
            self.nodes[node_k] = Node()
            self.nodes[node_k].assign(node_v)
        return
    
    def _get_cores_consumption_warnings(self):
        """Returns list of warning strings in case warnings are generated. 
        In case no warnings are generated the return value will be an empty list"""
        warnings = list()
        #Get the cores consumption
        (return_status, code) = get_ssh_connection(self.machine, self.user, self.password)
        if not return_status:
            return (return_status, code)
        ssh_conn = code
        command = 'mpstat -P ALL 1 1 | grep Average | egrep -v CPU\|all'
        (_, stdout, stderr) = ssh_conn.exec_command(command)
        error = stderr.read()
        if len(error) > 0:
            return (False, error)
    
        self.cores_consumption = dict()
        for line in stdout.readlines():
            cpu_usage_split = re.split('\t| *', line.rstrip('\n'))
            usage = 100 *(1 - float(cpu_usage_split[10]))
            if usage > 0:
                self.cores_consumption[int(cpu_usage_split[1])] = usage 
        ssh_conn.close()   
        #Check if any core marked as available in the nodes has cpu_usage > 0
        for _, node_v in self.nodes.iteritems():
            cores = node_v.processor.eligible_cores
            for cpu in cores:
                if len(cpu) > 1:
                    for core in cpu:
                        if core in self.cores_consumption:
                            warnings.append('Warning: Core '+str(core)+' is supposed to be idle but it is consuming '+str(self.cores_consumption[core])+'%')
                else:
                    if cpu in self.cores_consumption:
                        warnings.append('Warning: Core '+str(core)+' is supposed to be idle but it is consuming '+str(self.cores_consumption[cpu])+'%')
        
        return warnings
    
    def reserved_to_text(self):
        text = str()
        for VNFC_name, VNFC_reserved in self.reserved.iteritems():
            text += '    VNFC: '+str(VNFC_name)+'\n'
            text += VNFC_reserved.to_text()
                    
        return text
    
    def obtain_usage(self):
        resp = dict()
        #Iterate through nodes to get cores, eligible cores, memory and physical ports (save ports usage for next section)
        nodes = dict()
        ports_usage = dict()
        hugepage_size = dict()
        for node_k, node_v in self.nodes.iteritems():
            node = dict()
            ports_usage[node_k] = dict()
            eligible_cores = list()
            for pair in node_v.processor.eligible_cores:
                if isinstance(pair, list):
                    for element in pair:
                        eligible_cores.append(element)
                else:
                    eligible_cores.append(pair)
            node['cpus'] = {'cores':node_v.processor.cores,'eligible_cores':eligible_cores}
            node['memory'] = {'size':str(node_v.memory.node_size/(1024*1024*1024))+'GB','eligible':str(node_v.memory.eligible_memory/(1024*1024*1024))+'GB'}
            hugepage_size[node_k] = node_v.memory.hugepage_sz
            
            ports = dict()
            for nic in node_v.nic_list:
                for port in nic.ports.itervalues():
                    if port.enabled and not port.virtual: 
                        ports[port.name] = {'speed':str(port.speed/1000000000)+'G'}
#                         print '*************** ',port.name,'speed',port.speed 
                        ports_usage[node_k][port.name] = 100 - int(100*float(port.available_bw)/float(port.speed))
            node['ports'] = ports
            nodes[node_k] = node
        resp['RAD'] = nodes
        
        #Iterate through reserved section to get used cores, used memory and port usage
        cores = dict()
        memory = dict()
        #reserved_cores = list
        for node_k in self.nodes.iterkeys():
            if not node_k in cores:
                cores[node_k] = list()
                memory[node_k] = 0
            for _, reserved in self.reserved.iteritems():
                if node_k in reserved.node_reserved_resources:
                    node_v = reserved.node_reserved_resources[node_k]
                    cores[node_k].extend(node_v.reserved_cores)
                    memory[node_k] += node_v.reserved_hugepage_nr * hugepage_size[node_k]
                            
        occupation = dict()       
        for node_k in self.nodes.iterkeys():
            ports = dict()
            for name, usage in ports_usage[node_k].iteritems():
                ports[name] = {'occupied':str(usage)+'%'}
#             print '****************cores',cores
#             print '****************memory',memory
            occupation[node_k] = {'cores':cores[node_k],'memory':str(memory[node_k]/(1024*1024*1024))+'GB','ports':ports}
        resp['occupation'] = occupation
        
        return resp            
    
class RADreservedResources():
    def __init__(self):
        self.node_reserved_resources = dict()      #dict. keys are the RAD nodes id, values are NodeReservedResources
        self.mgmt_interface_pci = None             #pci in the VNF for the management interface
        self.image = None                          #Path in remote machine of the VNFC image
    
    def update(self,reserved):
        self.image = reserved.image
        self.mgmt_interface_pci = reserved.mgmt_interface_pci
        for k,v in reserved.node_reserved_resources.iteritems():
            if k in self.node_reserved_resources.keys():
                return (False, 'Duplicated node entry '+str(k)+' in reserved resources')
            self.node_reserved_resources[k]=v
            
        return (True, "")
    
    def to_text(self):
        text = '        image: '+str(self.image)+'\n'
        for node_id, node_reserved in self.node_reserved_resources.iteritems():
            text += '        Node ID: '+str(node_id)+'\n'
            text += node_reserved.to_text()
        return text

class NodeReservedResources():
    def __init__(self):
    #     reserved_shared_cores = None      #list. List of all cores that the VNFC needs in shared mode  #TODO Not used
    #     reserved_memory = None            #Integer. Amount of KiB needed by the VNFC #TODO. Not used since hugepages are used
        self.reserved_cores = list()             #list. List of all cores that the VNFC uses
        self.reserved_hugepage_nr = 0            #Integer. Number of hugepages needed by the VNFC 
        self.reserved_ports = dict()             #dict. The key is the physical port pci and the value the VNFC port description
        self.vlan_tags = dict()
        self.cpu_pinning = None
    
    def to_text(self):
        text = '            cores: '+str(self.reserved_cores)+'\n'
        text += '            cpu_pinning: '+str(self.cpu_pinning)+'\n'
        text += '            hugepages_nr: '+str(self.reserved_hugepage_nr)+'\n'
        for port_pci, port_description in self.reserved_ports.iteritems():
            text += '            port: '+str(port_pci)+'\n'
            text += port_description.to_text()
        return text
    
#     def update(self,reserved):
#         self.reserved_cores = list(reserved.reserved_cores)
#         self.reserved_hugepage_nr = reserved.reserved_hugepage_nr
#         self.reserved_ports = dict(reserved.reserved_ports)
#         self.cpu_pinning = list(reserved.cpu_pinning)