robot-systest/testsuite/heal_02-scale_vdu_healing.robot

*** Comments ***
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.


*** Settings ***
Documentation   [HEAL-02] Healing of scaled VDUs

Library   OperatingSystem
Library   String
Library   Collections
Library   Process
Library   SSHLibrary

Resource   ../lib/vnfd_lib.resource
Resource   ../lib/vnf_lib.resource
Resource   ../lib/nsd_lib.resource
Resource   ../lib/ns_lib.resource
Resource   ../lib/ns_operation_lib.resource
Resource   ../lib/ssh_lib.resource
Resource   ../lib/openstack_lib.resource

Test Tags   heal_02   cluster_heal   daily   regression

Suite Teardown   Run Keyword And Ignore Error   Suite Cleanup


*** Variables ***
# NS and VNF descriptor package folder and ids
${VNFD_VOLUMES_PKG}   several_volumes_vnf
${VNFD_VOLUMES_NAME}   several_volumes-vnf
${VDU_VOLUMES_NAME}   several_volumes-VM
${VNF_SEVERAL_INDEX}   several_volumes_vnf
${VNFD_MANUALSCALE_PKG}   manual_scale_vnf
${VNFD_MANUALSCALE_NAME}   manual_scale-vnf
${VDU_MANUALSCALE_NAME}   mgmtVM
${VNF_MANUALSCALE_INDEX}   manual_scale_vnf
${VNF_MANUALSCALE_SCALING_GROUP}   manual-scaling_mgmtVM
${VNF_MANUALSCALE_CLOUDINIT_FILE}   /root/helloworld.txt
${VNF_MANUALSCALE_DAY1_FILE}   /home/ubuntu/first-touch
${NSD_PKG}   volumes_healing_ns
${NSD_NAME}   volumes_healing-ns

# NS instance name and configuration
${FLAVOR_NAME_PREFIX}   osm.heal02
${NS_NAME}   heal_02
${NS_TIMEOUT}   6min
${SCALE_WAIT_TIME}   5min

# SSH keys and username to be used
${PUBLICKEY}   %{HOME}/.ssh/id_rsa.pub
${PRIVATEKEY}   %{HOME}/.ssh/id_rsa
${USERNAME}   ubuntu
${PASSWORD}   ${EMPTY}

${SUCCESS_RETURN_CODE}   0

@{VIM_VDUS}   @{EMPTY}
@{VIM_VOLUMES}   @{EMPTY}

# Variables to control time for healing and polling
${VNF_MAX_TIME_TO_BE_READY}   120sec
${VNF_POL_TIME}   30sec


*** Test Cases ***
Create VNF Descriptors
    [Documentation]   Upload VNF packages for the testsuite.
    Create VNFD   '%{PACKAGES_FOLDER}/${VNFD_MANUALSCALE_PKG}'
    Create VNFD   '%{PACKAGES_FOLDER}/${VNFD_VOLUMES_PKG}'

Create NS Descriptor
    [Documentation]   Upload NS package for the testsuite.
    Create NSD   '%{PACKAGES_FOLDER}/${NSD_PKG}'

Create Test Flavor
    [Documentation]   Create a flavor that will be used at NS instantiation time and save it as FLAVOR_ID.
    ${rand}=   Generate Random String   6   [NUMBERS]
    ${flavor_name}=   Catenate   SEPARATOR=_   ${FLAVOR_NAME_PREFIX}   ${rand}
    ${id}=   Create Flavor   ${flavor_name}   1   1024   10
    Set Suite Variable   ${FLAVOR_ID}   ${id}

Network Service Instance Test
    [Documentation]   Instantiate NS for the testsuite using the previously created flavor.
    ${ns_config}=   Set Variable   {vld: [ {name: mgmtnet, vim-network-name: %{VIM_MGMT_NET}}],vnf: [ {member-vnf-index: manual_scale_vnf, vdu: [{ id: mgmtVM, vim-flavor-id: ${FLAVOR_ID}}]}] }
    ${id}=   Create Network Service   ${NSD_NAME}   %{VIM_TARGET}   ${NS_NAME}   ${ns_config}   ${PUBLICKEY}   ${NS_TIMEOUT}
    Set Suite Variable   ${NS_ID}   ${id}

Get NS Id
    [Documentation]   Get NS identifier.
    [Tags]   cleanup
    ${variables}=   Get Variables
    IF   not "\${ns_id}" in "${variables}"
        ${id}=   Get Ns Id   ${NS_NAME}
        Set Suite Variable   ${NS_ID}   ${id}
    END

Scale Out Manual Scale VNF
    [Documentation]   Perform a manual scale-out operation of the manual-scale VNF.
    ${vnf_id}=   Get Vnf Id   ${NS_ID}   ${VNF_MANUALSCALE_INDEX}
    Set Suite Variable   ${VNF_MANUALSCALE_ID}   ${vnf_id}
    @{vdur_list}=   Get Vnf Vdur Names   ${VNF_MANUALSCALE_ID}
    ${vdurs}=   Get Length   ${vdur_list}
    Set Suite Variable   ${INITIAL_VDUR_COUNT}   ${vdurs}
    Execute Manual VNF Scale   ${NS_NAME}   ${VNF_MANUALSCALE_INDEX}   ${VNF_MANUALSCALE_SCALING_GROUP}   SCALE_OUT   ${SCALE_WAIT_TIME}
    @{vdur_list}=   Get Vnf Vdur Names   ${VNF_MANUALSCALE_ID}
    ${vdurs}=   Get Length   ${vdur_list}
    IF   ${vdurs} != ${INITIAL_VDUR_COUNT} + 1   Fail   msg=There is no new VDU records in the VNF after Scale Out

Get VIM Objects
    [Documentation]   Retrieve all VMs and volumes from the NS and stores them in VIM_VDUS and VIM_VOLUMES lists.
    Variable Should Exist   ${NS_ID}   msg=NS is not available
    @{vnf_id_list}=   Get Ns Vnf List   ${NS_ID}
    Log   ${vnf_id_list}
    FOR   ${vnf_id}   IN   @{vnf_id_list}
        Log   ${vnf_id}
        ${id}=   Get VNF VIM ID   ${vnf_id}
        @{vdu_ids}=   Split String   ${id}
        Append To List   ${VIM_VDUS}   @{vdu_ids}
    END
    FOR   ${vdu_id}   IN   @{VIM_VDUS}
        ${volumes_attached}=   Get Server Property   ${vdu_id}   volumes_attached
        ${match}=   Get Regexp Matches   ${volumes_attached}   '([0-9a-f\-]+)'   1
        IF   ${match} != @{EMPTY}
            IF   not "${match}[0]" in "@{VIM_VOLUMES}"
                Append To List   ${VIM_VOLUMES}   ${match}[0]
            END
        END
    END
    Log Many   @{VIM_VDUS}
    Log Many   @{VIM_VOLUMES}

Get Manual Scale VNF Info
    [Documentation]   Get VDU ID and IP addresses of the manual scale VNF and stores them in VDU_MANUALSCALE_IDS and MANUALSCALE_IP_LIST.
    Variable Should Exist   ${NS_ID}   msg=NS is not available
    ${variables}=   Get Variables
    IF   not "\${VNF_MANUALSCALE_ID}" in "${variables}"
        ${vnf_id}=   Get Vnf Id   ${NS_ID}   ${VNF_MANUALSCALE_INDEX}
        Set Suite Variable   ${VNF_MANUALSCALE_ID}   ${vnf_id}
    END
    ${id}=   Get VNF VIM ID   ${VNF_MANUALSCALE_ID}
    @{vdu_manualscale_ids}=   Split String   ${id}
    Set Suite Variable   @{VDU_MANUALSCALE_IDS}   @{vdu_manualscale_ids}
    Log   ${VDU_MANUALSCALE_IDS}[1]
    @{manualscale_ip_list}=   Get Vnf Vdur IPs   ${VNF_MANUALSCALE_ID}
    Set Suite Variable   @{MANUALSCALE_IP_LIST}   @{manualscale_ip_list}
    ${ip}=   Get Vdu Attribute   ${VNF_MANUALSCALE_ID}   ip-address   1
    Set Suite Variable   ${HEALED_VDU_MGMT_IP}   ${ip}

Check Day0 And Day1 In VDU Before Halting VM
    [Documentation]   Check that the VDU is accessible via SSH in its mgmt IP address.
    ...               It also checks if day-0 worked and a remote file has been created in the VDU.
    ...               It also checks if day-1 worked and a remote file has been created in the VDU.
    ${ip}=   Get Vdu Attribute   ${VNF_MANUALSCALE_ID}   ip-address   1
    Variable Should Exist   ${HEALED_VDU_MGMT_IP}   msg=IP address of the healed VDU is not available
    Wait Until Keyword Succeeds   ${VNF_MAX_TIME_TO_BE_READY}   ${VNF_POL_TIME}   Test SSH Connection   ${HEALED_VDU_MGMT_IP}   ${USERNAME}   ${PASSWORD}   ${PRIVATEKEY}
    ${stdout}=   Execute Remote Command Check Rc Return Output   ${ip}   ${USERNAME}   ${PASSWORD}   ${PRIVATEKEY}   sudo ls ${VNF_MANUALSCALE_CLOUDINIT_FILE}
    Log   ${stdout}
    # Check If Remote File Exists   ${HEALED_VDU_MGMT_IP}   ${USERNAME}   ${PASSWORD}   ${PRIVATEKEY}   ${VNF_MANUALSCALE_DAY1_FILE}

Halt Manual Scale VDU
    [Documentation]   Halt one of the VM of the Manual Scale VNF.
    Variable Should Exist   @{VDU_MANUALSCALE_IDS}   msg=VDU is not available
    Halt Server   ${VDU_MANUALSCALE_IDS}[1]
    Sleep   15

Heal Manual Scale VDU
    [Documentation]   Heal manually via OSM commands all stopped VMs . They should be started again.
    Variable Should Exist   ${VNF_MANUALSCALE_ID}   msg=VNF is not available
    Heal Network Service   ${NS_ID}   --vnf ${VNF_MANUALSCALE_ID} --cause "Heal VM of manual_scale_vnf" --vdu ${VDU_MANUALSCALE_NAME} --count-index 1 --run-day1

Check VNF After Healing
    [Documentation]   Check that the IDs of the VM and volumes have not changed after healing.
    Variable Should Exist   ${VNF_MANUALSCALE_ID}   msg=VNF is not available
    @{ip_list}=   Get Vnf Vdur IPs   ${VNF_MANUALSCALE_ID}
    Should Be Equal   ${ip_list}   ${MANUALSCALE_IP_LIST}   IP addresses have changed after healing
    ${id}=   Get VNF VIM ID   ${VNF_MANUALSCALE_ID}
    @{ids}=   Split String   ${id}
    Should Be Equal   ${VDU_MANUALSCALE_IDS}[0]   ${ids}[0]   VDU[0] id has changed after healing
    Should Not Be Equal   ${VDU_MANUALSCALE_IDS}[1]   ${ids}[1]   VDU[1] id has not changed after healing
    Should Be Equal   ${VDU_MANUALSCALE_IDS}[2]   ${ids}[2]   VDU[2] id has changed after healing
    ${vim_info}=   Get Vdu Attribute   ${VNF_MANUALSCALE_ID}   vim_info   1
    Should Contain   ${vim_info}   id: ${FLAVOR_ID}   msg=Flavor ID is incorrect
    ${ip}=   Get Vdu Attribute   ${VNF_MANUALSCALE_ID}   ip-address   1
    Should Be Equal   ${HEALED_VDU_MGMT_IP}   ${ip}

Check Day0 And Day1 In VDU After Healing
    [Documentation]   Check that the healed VDU is accessible via SSH in its mgmt IP address.
    ...               It also checks if day-0 worked after healing and a remote file has been created in the VDU.
    ...               It also checks if day-1 worked after healing and a remote file has been created in the VDU.
    Variable Should Exist   ${HEALED_VDU_MGMT_IP}   msg=IP address of the healed VDU is not available
    Wait Until Keyword Succeeds   ${VNF_MAX_TIME_TO_BE_READY}   ${VNF_POL_TIME}   Test SSH Connection   ${HEALED_VDU_MGMT_IP}   ${USERNAME}   ${PASSWORD}   ${PRIVATEKEY}
    ${stdout}=   Execute Remote Command Check Rc Return Output   ${HEALED_VDU_MGMT_IP}   ${USERNAME}   ${PASSWORD}   ${PRIVATEKEY}   sudo ls ${VNF_MANUALSCALE_CLOUDINIT_FILE}
    Log   ${stdout}
    # Check If Remote File Exists   ${HEALED_VDU_MGMT_IP}   ${USERNAME}   ${PASSWORD}   ${PRIVATEKEY}   ${VNF_MANUALSCALE_DAY1_FILE}

Update VIM Objects
    [Documentation]   Retrieve all VMs and volumes from the NS and stores them in VIM_VDUS and VIM_VOLUMES lists.
    ...               This is done again to guarantee that all objects are cleaned in the VIM in case the heal operation
    ...               added new objects.
    Variable Should Exist   ${NS_ID}   msg=NS is not available
    @{vdu_updated}=   Create List
    @{vnf_id_list}=   Get Ns Vnf List   ${NS_ID}
    FOR   ${vnf_id}   IN   @{vnf_id_list}
        ${id}=   Get VNF VIM ID   ${vnf_id}
        @{vdu_ids}=   Split String   ${id}
        Append To List   ${vdu_updated}   @{vdu_ids}
        FOR   ${id}   IN   @{vdu_ids}
            IF   not "${id}" in "@{VIM_VDUS}"
                Append To List   ${VIM_VDUS}   ${id}
            END
        END
    END
    FOR   ${vdu_id}   IN   @{vdu_updated}
        ${volumes_attached}=   Get Server Property   ${vdu_id}   volumes_attached
        ${match}=   Get Regexp Matches   ${volumes_attached}   '([0-9a-f\-]+)'   1
        IF   ${match} != @{EMPTY}
            IF   not "${match}[0]" in "@{VIM_VOLUMES}"
                Append To List   ${VIM_VOLUMES}   ${match}[0]
            END
        END
    END
    Log Many   @{VIM_VDUS}
    Log Many   @{VIM_VOLUMES}

Delete NS Instance
    [Documentation]   Delete NS instance.
    [Tags]   cleanup
    Delete NS   ${NS_NAME}

Delete NS Descriptor
    [Documentation]   Delete NS package from OSM.
    [Tags]   cleanup
    Delete NSD   ${NSD_NAME}

Delete VNF Descriptors
    [Documentation]   Delete VNF packages from OSM.
    [Tags]   cleanup
    Delete VNFD   ${VNFD_VOLUMES_NAME}
    Delete VNFD   ${VNFD_MANUALSCALE_NAME}

Delete Remaining Objects In VIM
    [Documentation]   Delete any remaining objects (volumes, VMs, etc.) in the VIM.
    [Tags]   cleanup
    Delete Objects In VIM


*** Keywords ***
Suite Cleanup
    [Documentation]   Test Suit Cleanup: Deleting Descriptor, instance and vim
    Run Keyword If Any Tests Failed   Delete NS   ${NS_NAME}
    Run Keyword If Any Tests Failed   Delete NSD   ${NSD_NAME}
    Run Keyword If Any Tests Failed   Delete VNFD   ${VNFD_VOLUMES_NAME}
    Run Keyword If Any Tests Failed   Delete VNFD   ${VNFD_MANUALSCALE_NAME}
    Run Keyword If Any Tests Failed   Delete Objects In VIM

Delete Objects In VIM
    [Documentation]   Clean up remaining VMs and volumes directly from the VIM.
    Delete Flavor   ${FLAVOR_ID}
    ${error}=   Set Variable   0
    FOR   ${vol_id}   IN   @{VIM_VOLUMES}
        Log   Checking if volume ${vol_id} is still in VIM
        ${exists}=   Check If Volume Exists   ${vol_id}
        IF   ${exists}
            ${error}=   Set Variable   1
            Log   Deleting volume ${vol_id}
            Run Keyword And Ignore Error   Delete Volume   ${vol_id}
        END
    END
    FOR   ${vdu_id}   IN   @{VIM_VDUS}
        Log   Checking if server ${vdu_id} is still in VIM
        ${status}=   Run Keyword And Ignore Error   Get Server Property   ${vdu_id}   id
        Log   ${status}[0]
        IF   '${status}[0]' == 'PASS'
            ${error}=   Set Variable   1
            Log   Deleting server ${vdu_id}
            Run Keyword And Ignore Error   Delete Server   ${vdu_id}
        END
    END
    IF   ${error}==1
        Fail   Some objects created by test were not deleted in VIM
    END