*** Comments ***
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.


*** Settings ***
Documentation   [HEAL-04] Autohealing of NS

Library   OperatingSystem
Library   String
Library   Collections
Library   Process
Library   SSHLibrary

Resource   ../lib/vnfd_lib.resource
Resource   ../lib/vnf_lib.resource
Resource   ../lib/nsd_lib.resource
Resource   ../lib/ns_lib.resource
Resource   ../lib/ns_operation_lib.resource
Resource   ../lib/ssh_lib.resource
Resource   ../lib/openstack_lib.resource

Test Tags   heal_04   cluster_heal   daily   regression

Suite Teardown   Run Keyword And Ignore Error   Suite Cleanup


*** Variables ***
# NS and VNF descriptor package folder and ids
${VNFD_PKG}   autoheal_vnf
${VNFD_NAME}   autoheal_vnfd
${VDU_NAME}   autoheal_vnfd-VM
${VNF_INDEX}   autoheal-basic-1
${NSD_PKG}   autoheal_ns
${NSD_NAME}   autoheal_nsd
# NS instance name and configuration
${NS_NAME}   heal_04
${NS_CONFIG}   {vld: [ {name: mgmt, vim-network-name: %{VIM_MGMT_NET}} ] }

# SSH keys and username to be used
${PUBLICKEY}   %{HOME}/.ssh/id_rsa.pub
${PRIVATEKEY}   %{HOME}/.ssh/id_rsa
${USERNAME}   ubuntu
${PASSWORD}   ${EMPTY}

${SUCCESS_RETURN_CODE}   0

# Healing wait time
${HEALING_POL_TIME}   15sec
${HEALING_MAX_WAIT_TIME}   10m

@{VIM_VDUS}   @{EMPTY}


*** Test Cases ***
Create VNF Descriptors
    [Documentation]   Upload VNF package for the testsuite.
    Create VNFD   '%{PACKAGES_FOLDER}/${VNFD_PKG}'

Create NS Descriptor
    [Documentation]   Upload NS package for the testsuite.
    Create NSD   '%{PACKAGES_FOLDER}/${NSD_PKG}'

Network Service Instance Test
    [Documentation]   Instantiate NS for the testsuite.
    ${id}=   Create Network Service   ${NSD_NAME}   %{VIM_TARGET}   ${NS_NAME}   ${NS_CONFIG}   ${PUBLICKEY}
    Set Suite Variable   ${NS_ID}   ${id}

Get NS Id
    [Documentation]   Get NS identifier and stores as suite variable to be used later on.
    [Tags]   cleanup
    ${variables}=   Get Variables
    IF   not "\${ns_id}" in "${variables}"
        ${id}=   Get Ns Id   ${NS_NAME}
        Set Suite Variable   ${NS_ID}   ${id}
    END

Get VIM Objects
    [Documentation]   Retrieve all VMs and volumes from the NS and stores them in VIM_VDUS and VIM_VOLUMES lists.
    Variable Should Exist   ${NS_ID}   msg=NS is not available
    @{vnf_id_list}=   Get Ns Vnf List   ${NS_ID}
    Log   ${vnf_id_list}
    FOR   ${vnf_id}   IN   @{vnf_id_list}
        Log   ${vnf_id}
        ${id}=   Get VNF VIM ID   ${vnf_id}
        @{vdu_ids}=   Split String   ${id}
        Append To List   ${VIM_VDUS}   @{vdu_ids}
    END
    Log Many   @{VIM_VDUS}

Get VNF Info
    [Documentation]   Get VDU ID and IP addresses of the VNF and stores them to be used later on.
    Variable Should Exist   ${NS_ID}   msg=NS is not available
    ${ip_addr}=   Get Vnf Management Ip Address   ${NS_ID}   ${VNF_INDEX}
    Log   ${ip_addr}
    Set Suite Variable   ${VNF_IP_ADDR}   ${ip_addr}
    ${vnf_id}=   Get Vnf Id   ${NS_ID}   ${VNF_INDEX}
    Set Suite Variable   ${VNF_AUTOHEAL_ID}   ${vnf_id}
    ${id}=   Get VNF VIM ID   ${vnf_id}
    Set Suite Variable   ${VDU_AUTOHEAL_ID}   ${id}
    Log   ${VDU_AUTOHEAL_ID}
    @{AUTOHEAL_IP_LIST}=   Get Vnf Vdur IPs   ${VNF_AUTOHEAL_ID}
    Set Suite Variable   @{AUTOHEAL_IP_LIST}   @{AUTOHEAL_IP_LIST}
    Log   @{AUTOHEAL_IP_LIST}

Stop Autoheal VDU
    [Documentation]   Stop one VM of the VNF.
    Variable Should Exist   ${VDU_AUTOHEAL_ID}   msg=VDU is not available
    Halt Server   ${VDU_AUTOHEAL_ID}
    Sleep   30

Wait For Autohealing To Be Completed
    [Documentation]   Wait for auto-healing to be completed. OSM will detect that the VM is down and will re-create it.
    ${healing_max_wait_time}=   Convert Time   ${HEALING_MAX_WAIT_TIME}   result_format=number
    ${healing_max_wait_time}=   Evaluate   ${healing_max_wait_time} * ${VIM_TIMEOUT_MULTIPLIER}
    Wait Until Keyword Succeeds   ${healing_max_wait_time}   ${HEALING_POL_TIME}   Get Operations By Type   ${NS_ID}   heal
    ${stdout}=   Get Operations By Type   ${NS_ID}   heal
    Wait Until Keyword Succeeds   ${healing_max_wait_time}   ${HEALING_POL_TIME}   Check For NS Operation Ended   ${stdout}
    Check For NS Operation Completed   ${stdout}

Check VNF After Healing
    [Documentation]   Check that the ID of the VM and the IP addresses have not changed after healing.
    Variable Should Exist   ${VNF_AUTOHEAL_ID}   msg=VNF is not available
    @{ip_list}=   Get Vnf Vdur IPs   ${VNF_AUTOHEAL_ID}
    Log   @{ip_list}
    Should Be Equal   ${ip_list}   ${AUTOHEAL_IP_LIST}   IP addresses have changed after healing
    ${id}=   Get VNF VIM ID   ${VNF_AUTOHEAL_ID}
    Log   ${id}
    Should Not Be Equal   ${id}   ${VDU_AUTOHEAL_ID}   VDU id has not changed after healing

Update VIM Objects
    [Documentation]   Retrieve all VMs from the NS and stores them in VIM_VDUS.
    ...               This is done again to guarantee that all objects are cleaned in the VIM in case the heal operation
    ...               added new objects.
    Variable Should Exist   ${NS_ID}   msg=NS is not available
    @{vnf_id_list}=   Get Ns Vnf List   ${NS_ID}
    FOR   ${vnf_id}   IN   @{vnf_id_list}
        ${id}=   Get VNF VIM ID   ${vnf_id}
        @{vdu_ids}=   Split String   ${id}
        FOR   ${id}   IN   @{vdu_ids}
            IF   not "${id}" in "@{VIM_VDUS}"
                Append To List   ${VIM_VDUS}   ${id}
            END
        END
    END
    Log Many   @{VIM_VDUS}

Delete NS Instance
    [Documentation]   Delete NS instance.
    [Tags]   cleanup
    Delete NS   ${NS_NAME}

Delete NS Descriptor
    [Documentation]   Delete NS package from OSM.
    [Tags]   cleanup
    Delete NSD   ${NSD_NAME}

Delete VNF Descriptors
    [Documentation]   Delete VNF package from OSM.
    [Tags]   cleanup
    Delete VNFD   ${VNFD_NAME}

Delete Objects In VIM
    [Documentation]   Delete any remaining objects (volumes, VMs, etc.) in the VIM.
    [Tags]   cleanup
    ${error}=   Set Variable   0
    FOR   ${vdu_id}   IN   @{VIM_VDUS}
        Log   Checking if server ${vdu_id} is still in VIM
        ${status}=   Run Keyword And Ignore Error   Get Server Property   ${vdu_id}   id
        Log   ${status}[0]
        IF   '${status}[0]' == 'PASS'
            ${error}=   Set Variable   1
            Log   Deleting server ${vdu_id}
            Run Keyword And Ignore Error   Delete Server   ${vdu_id}
        END
    END
    IF   ${error}==1
        Fail   Some objects created by test were not deleted in VIM
    END


*** Keywords ***
Suite Cleanup
    [Documentation]   Test Suit Cleanup: Deleting Descriptor, instance and vim
    Run Keyword If Any Tests Failed   Delete NS   ${NS_NAME}
    Run Keyword If Any Tests Failed   Delete NSD   ${NSD_NAME}
    Run Keyword If Any Tests Failed   Delete VNFD   ${VNFD_NAME}