From 845c2ea1f8d825af847dbb94ac14e77eb8b3c169 Mon Sep 17 00:00:00 2001 From: aguilard Date: Fri, 8 Apr 2022 09:36:03 +0000 Subject: [PATCH] Feature 10909: Robot tests for healing Change-Id: I86bab7a72baf6f4accec74e2d54a7b4dfdbe6c38 Signed-off-by: aguilard --- robot-systest/lib/ns_lib.robot | 60 +++++- robot-systest/lib/openstack_lib.robot | 11 + robot-systest/lib/vnf_lib.robot | 23 ++ .../heal_01-volume_vdu_healing.robot | 187 +++++++++++++++++ .../testsuite/heal_02-scale_vdu_healing.robot | 176 ++++++++++++++++ .../testsuite/heal_03-multiple_healing.robot | 198 ++++++++++++++++++ 6 files changed, 651 insertions(+), 4 deletions(-) create mode 100644 robot-systest/testsuite/heal_01-volume_vdu_healing.robot create mode 100644 robot-systest/testsuite/heal_02-scale_vdu_healing.robot create mode 100644 robot-systest/testsuite/heal_03-multiple_healing.robot diff --git a/robot-systest/lib/ns_lib.robot b/robot-systest/lib/ns_lib.robot index 7c2b794..3cd863e 100644 --- a/robot-systest/lib/ns_lib.robot +++ b/robot-systest/lib/ns_lib.robot @@ -23,6 +23,7 @@ ${ns_delete_pol_time} 15sec ${ns_action_max_wait_time} 1min ${ns_action_pol_time} 15sec ${vnf_scale_pol_time} 15sec +${healing_pol_time} 15sec ${vim_timeout_multiplier} %{OSM_VIM_MULTIPLIER_TIMEOUT=1.0} @@ -196,7 +197,8 @@ Execute NS Action ${osm_ns_action_command}= Catenate ${osm_ns_action_command} ${ns_name} ${rc} ${stdout}= Run and Return RC and Output ${osm_ns_action_command} Should Be Equal As Integers ${rc} ${success_return_code} msg=${stdout} values=False - Wait Until Keyword Succeeds ${ns_action_max_wait_time} ${ns_action_pol_time} Check For NS Operation Completed ${stdout} + Wait Until Keyword Succeeds ${ns_action_max_wait_time} ${ns_action_pol_time} Check For NS Operation Ended ${stdout} + Check For NS Operation Completed ${stdout} [Return] ${stdout} @@ -221,7 +223,8 @@ Execute NS K8s Action ${osm_ns_action_command}= Catenate ${osm_ns_action_command} ${ns_name} ${rc} ${stdout}= Run and Return RC and Output ${osm_ns_action_command} Should Be Equal As Integers ${rc} ${success_return_code} msg=${stdout} values=False - Wait Until Keyword Succeeds ${ns_action_max_wait_time} ${ns_action_pol_time} Check For NS Operation Completed ${stdout} + Wait Until Keyword Succeeds ${ns_action_max_wait_time} ${ns_action_pol_time} Check For NS Operation Ended ${stdout} + Check For NS Operation Completed ${stdout} [Return] ${stdout} @@ -241,13 +244,32 @@ Execute Manual VNF Scale ${osm_vnf_scale_command}= Catenate ${osm_vnf_scale_command} ${ns_name} ${vnf_member_index} ${rc} ${stdout}= Run and Return RC and Output ${osm_vnf_scale_command} Should Be Equal As Integers ${rc} ${success_return_code} msg=${stdout} values=False - Wait Until Keyword Succeeds ${vnf_scale_max_wait_time} ${vnf_scale_pol_time} Check For NS Operation Completed ${stdout} + Wait Until Keyword Succeeds ${vnf_scale_max_wait_time} ${vnf_scale_pol_time} Check For NS Operation Ended ${stdout} + Check For NS Operation Completed ${stdout} + [Return] ${stdout} + + +Heal Network Service + [Documentation] Execute healing operation over one NS. + ... Return the ID of the operation associated to the executed healing action. + + [Arguments] ${ns_name} ${params} ${healing_max_wait_time}=10m + + Should Not Be Empty ${ns_name} + Should Not Be Empty ${params} + ${healing_max_wait_time}= Convert Time ${healing_max_wait_time} result_format=number + ${healing_max_wait_time}= Evaluate ${healing_max_wait_time} * ${vim_timeout_multiplier} + ${rc} ${stdout}= Run and Return RC and Output osm ns-heal ${ns_name} ${params} + Should Be Equal As Integers ${rc} ${success_return_code} msg=${stdout} values=False + Wait Until Keyword Succeeds ${healing_max_wait_time} ${healing_pol_time} Check For NS Operation Ended ${stdout} + Check For NS Operation Completed ${stdout} [Return] ${stdout} Get Operations List [Arguments] ${ns_name} + Should Not Be Empty ${ns_name} ${rc} ${stdout}= Run and Return RC and Output osm ns-op-list ${ns_name} log ${stdout} log ${rc} @@ -259,10 +281,35 @@ Check For NS Operation Completed [Arguments] ${ns_operation_id} + Should Not Be Empty ${ns_operation_id} + ${rc} ${stdout}= Run and Return RC and Output osm ns-op-show ${ns_operation_id} --literal | yq .operationState | tr -d \\" + log ${stdout} + Should Be Equal As Integers ${rc} ${success_return_code} + Should Contain ${stdout} COMPLETED msg=The ns-action with id ${ns_operation_id} was not completed values=False + + +Check For NS Operation Failed + [Documentation] Check wheter the status of the desired operation is "FAILED" or not. + + [Arguments] ${ns_operation_id} + + Should Not Be Empty ${ns_operation_id} ${rc} ${stdout}= Run and Return RC and Output osm ns-op-show ${ns_operation_id} --literal | yq .operationState | tr -d \\" log ${stdout} Should Be Equal As Integers ${rc} ${success_return_code} - Should Contain ${stdout} COMPLETED msg=Timeout waiting for ns-action with id ${ns_operation_id} values=False + Should Contain ${stdout} FAILED msg=Timeout waiting for ns-action with id ${ns_operation_id} values=False + + +Check For NS Operation Ended + [Documentation] Check wheter the status of the desired operation is "FAILED" or "COMPLETED". + + [Arguments] ${ns_operation_id} + + Should Not Be Empty ${ns_operation_id} + ${rc} ${stdout}= Run and Return RC and Output osm ns-op-show ${ns_operation_id} --literal | yq .operationState | tr -d \\" + log ${stdout} + Should Be Equal As Integers ${rc} ${success_return_code} + Should Contain Any ${stdout} FAILED COMPLETED msg=Timeout waiting for ns-action with id ${ns_operation_id} values=False Get Ns Vnfr Ids @@ -270,6 +317,7 @@ Get Ns Vnfr Ids [Arguments] ${ns_id} + Should Not Be Empty ${ns_id} ${rc} ${stdout}= Run and Return RC and Output osm vnf-list | grep ${ns_id} | awk '{print $2}' 2>&1 Should Be Equal As Integers ${rc} ${success_return_code} msg=${stdout} values=False @{vdur} = Split String ${stdout} @@ -281,6 +329,7 @@ Get Vnf Vdur Names [Arguments] ${vnf_id} + Should Not Be Empty ${vnf_id} ${rc} ${stdout}= Run and Return RC and Output osm vnf-show ${vnf_id} --literal | yq .vdur[].name | tr -d \\" Should Be Equal As Integers ${rc} ${success_return_code} msg=${stdout} values=False @{vdur} = Split String ${stdout} @@ -292,6 +341,8 @@ Get Vnf Kdu Replica Count [Arguments] ${vnf_id} ${kdu_name} + Should Not Be Empty ${vnf_id} + Should Not Be Empty ${kdu_name} ${rc} ${stdout}= Run and Return RC and Output osm vnf-show ${vnf_id} --kdu ${kdu_name} | yq .config.replicaCount | tr -d \\" Should Be Equal As Integers ${rc} ${success_return_code} msg=${stdout} values=False ${return} = Set Variable If '${stdout}' == 'null' ${EMPTY} ${stdout} @@ -303,6 +354,7 @@ Get Application Names [Arguments] ${ns_name} + Should Not Be Empty ${ns_name} ${rc} ${stdout}= Run and Return RC and Output osm ns-show ${ns_name} --literal | yq ._admin.deployed.VCA[].application | tr -d \\" Should Be Equal As Integers ${rc} ${success_return_code} msg=${stdout} values=False @{app_names} = Split String ${stdout} diff --git a/robot-systest/lib/openstack_lib.robot b/robot-systest/lib/openstack_lib.robot index d544bb5..36e3f1d 100644 --- a/robot-systest/lib/openstack_lib.robot +++ b/robot-systest/lib/openstack_lib.robot @@ -103,3 +103,14 @@ Delete Volume log ${stdout} Should Be Equal As Integers ${rc} ${success_return_code} [Return] ${stdout} + + +Stop Server + [Documentation] Stop a server + [Arguments] ${server_id} + + Should Not Be Empty ${server_id} + ${rc} ${stdout}= Run and Return RC and Output openstack server stop ${server_id} + log ${stdout} + Should Be Equal As Integers ${rc} ${success_return_code} + [Return] ${stdout} diff --git a/robot-systest/lib/vnf_lib.robot b/robot-systest/lib/vnf_lib.robot index 7b7d86d..d58c3dd 100644 --- a/robot-systest/lib/vnf_lib.robot +++ b/robot-systest/lib/vnf_lib.robot @@ -49,3 +49,26 @@ Get VDU VIM Id Should Be Equal As Integers ${rc} ${success_return_code} msg=${vm_vim_id} values=False [Return] ${vm_vim_id} + +Get Vnf Vdur IPs + [Documentation] Return a list with the IP addresses of the VDU records of a VNF instance. + + [Arguments] ${vnf_id} + + Should Not Be Empty ${vnf_id} + ${rc} ${stdout}= Run and Return RC and Output osm vnf-show ${vnf_id} --literal | yq '.vdur[].interfaces[]."ip-address"' | tr -d \\" + Should Be Equal As Integers ${rc} ${success_return_code} msg=${stdout} values=False + @{ip} = Split String ${stdout} + [Return] @{ip} + + +Get Vdu Attribute + [Documentation] Return an attribute from VDU records selected by count_index of a VNF instance. + + [Arguments] ${vnf_id} ${attribute} ${count_index}=0 + + Should Not Be Empty ${vnf_id} + Should Not Be Empty ${attribute} + ${rc} ${stdout}= Run and Return RC and Output osm vnf-show ${vnf_id} --literal | yq '.vdur[] | select(."count-index" == ${count_index})' | yq '."${attribute}"' | tr -d \\" + Should Be Equal As Integers ${rc} ${success_return_code} msg=${stdout} values=False + [Return] ${stdout} diff --git a/robot-systest/testsuite/heal_01-volume_vdu_healing.robot b/robot-systest/testsuite/heal_01-volume_vdu_healing.robot new file mode 100644 index 0000000..774a502 --- /dev/null +++ b/robot-systest/testsuite/heal_01-volume_vdu_healing.robot @@ -0,0 +1,187 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation [HEAL-01] Healing of a multi-volume VDU + +Library OperatingSystem +Library String +Library Collections +Library Process +Library SSHLibrary + +Resource %{ROBOT_DEVOPS_FOLDER}/lib/vnfd_lib.robot +Resource %{ROBOT_DEVOPS_FOLDER}/lib/vnf_lib.robot +Resource %{ROBOT_DEVOPS_FOLDER}/lib/nsd_lib.robot +Resource %{ROBOT_DEVOPS_FOLDER}/lib/ns_lib.robot +Resource %{ROBOT_DEVOPS_FOLDER}/lib/ssh_lib.robot +Resource %{ROBOT_DEVOPS_FOLDER}/lib/openstack_lib.robot + +Force Tags heal_01 cluster_main daily regression + + +Suite Teardown Run Keyword And Ignore Error Suite Cleanup + + +*** Variables *** +# NS and VNF descriptor package folder and ids +${vnfd_volumes_pkg} several_volumes_vnf +${vnfd_volumes_name} several_volumes-vnf +${vdu_volumes_name} several_volumes-VM +${vnf_several_index} several_volumes_vnf +${vnfd_charm_pkg} charm-packages/native_manual_scale_charm_vnf +${vnfd_charm_name} native_manual_scale_charm-vnf +${nsd_pkg} volumes_nativecharm_ns +${nsd_name} volumes_nativecharm-ns + +# NS instance name and configuration +${ns_name} heal_01 +${ns_config} {vld: [ {name: mgmtnet, vim-network-name: %{VIM_MGMT_NET}} ] } + +# SSH keys and username to be used +${publickey} %{HOME}/.ssh/id_rsa.pub +${privatekey} %{HOME}/.ssh/id_rsa +${username} ubuntu +${password} ${EMPTY} + +${success_return_code} 0 + + +*** Test Cases *** +Create VNF Descriptors + [Tags] prepare + Create VNFD '%{PACKAGES_FOLDER}/${vnfd_charm_pkg}' + Create VNFD '%{PACKAGES_FOLDER}/${vnfd_volumes_pkg}' + + +Create NS Descriptor + [Tags] prepare + Create NSD '%{PACKAGES_FOLDER}/${nsd_pkg}' + + +Network Service Instance Test + [Tags] prepare + ${id}= Create Network Service ${nsd_name} %{VIM_TARGET} ${ns_name} ${ns_config} ${publickey} + Set Suite Variable ${ns_id} ${id} + + +Get NS Id + [Tags] verify cleanup + ${variables} Get Variables + IF not "\${ns_id}" in "${variables}" + ${id}= Get Ns Id ${ns_name} + Set Suite Variable ${ns_id} ${id} + END + + +Get Volume VNF Info + [Tags] verify + Variable Should Exist ${ns_id} msg=NS is not available + ${ip_addr}= Get Vnf Management Ip Address ${ns_id} ${vnf_several_index} + log ${ip_addr} + Set Suite Variable ${vnf_volumes_ip_addr} ${ip_addr} + + ${vnf_id}= Get Vnf Id ${ns_id} ${vnf_several_index} + Set Suite Variable ${vnf_volumes_id} ${vnf_id} + ${id}= Get VNF VIM ID ${vnf_id} + Set Suite Variable ${vdu_volumes_id} ${id} + log ${vdu_volumes_id} + + @{volumes_ip_list}= Get Vnf Vdur IPs ${vnf_volumes_id} + Set Suite Variable @{volumes_ip_list} @{volumes_ip_list} + log @{volumes_ip_list} + + +Get Volumes Info + [Tags] verify + ${rc} ${stdout}= Run and Return RC and Output osm vnfpkg-show ${vnfd_volumes_name} --literal | yq '.vdu[0]."virtual-storage-desc" | length' + Should Be Equal As Integers ${rc} ${success_return_code} msg=${stdout} values=False + ${num_virtual_storage}= Convert To Integer ${stdout} + Set Suite Variable ${vnf_num_volumes} ${num_virtual_storage} + log ${vnf_num_volumes} + ${volumes_attached}= Get Server Property ${vdu_volumes_id} volumes_attached + ${match}= Get Regexp Matches ${volumes_attached} '([0-9a-f\-]+)' 1 + Set Suite Variable ${volume_id} ${match}[0] + + +Check VDU Disks + [Tags] verify + Variable Should Exist ${vnf_volumes_ip_addr} msg=VNF is not available + Sleep 30 seconds Wait for SSH daemon to be up + ${stdout}= Execute Remote Command Check Rc Return Output ${vnf_volumes_ip_addr} ${username} ${password} ${privatekey} sudo lsblk -l + log ${stdout} + ${lines}= Get Lines Containing String ${stdout} disk + ${num_lines}= Get Line Count ${lines} + Run Keyword If ${num_lines} < ${vnf_num_volumes} Fail msg=Number of disks (${num_lines}) is less than specified in VDU (${vnf_num_volumes}) + + +Stop Persistent Volume VDU + [Tags] verify + Variable Should Exist ${vdu_volumes_id} msg=VDU is not available + Stop Server ${vdu_volumes_id} + Sleep 20 + + +Heal Persistent Volume VDU + [Tags] verify + Variable Should Exist ${vnf_volumes_id} msg=VNF is not available + Heal Network Service ${ns_id} --vnf ${vnf_volumes_id} --cause "Heal VM of volumes_vnf" --vdu ${vdu_volumes_name} + + +Check VNF After Healing + [Tags] verify + Variable Should Exist ${vnf_volumes_id} msg=VNF is not available + + @{ip_list}= Get Vnf Vdur IPs ${vnf_volumes_id} + log @{ip_list} + Should Be Equal ${ip_list} ${volumes_ip_list} IP addresses have changed after healing + + ${id}= Get VNF VIM ID ${vnf_volumes_id} + log ${id} + Should Not Be Equal ${id} ${vdu_volumes_id} VDU id has not changed after healing + + ${volumes_attached}= Get Server Property ${id} volumes_attached + ${match}= Get Regexp Matches ${volumes_attached} '([0-9a-f\-]+)' 1 + Should Be Equal ${match}[0] ${volume_id} Volume id has changed after healing + + Sleep 30 seconds Wait for SSH daemon to be up + ${stdout}= Execute Remote Command Check Rc Return Output ${vnf_volumes_ip_addr} ${username} ${password} ${privatekey} sudo lsblk -l + log ${stdout} + ${lines}= Get Lines Containing String ${stdout} disk + ${num_lines}= Get Line Count ${lines} + Run Keyword If ${num_lines} < ${vnf_num_volumes} Fail msg=Number of disks (${num_lines}) is less than specified in VDU (${vnf_num_volumes}) + + +Delete NS Instance + [Tags] cleanup + Delete NS ${ns_name} + + +Delete NS Descriptor + [Tags] cleanup + Delete NSD ${nsd_name} + + +Delete VNF Descriptors + [Tags] cleanup + Delete VNFD ${vnfd_volumes_name} + Delete VNFD ${vnfd_charm_name} + + +*** Keywords *** +Suite Cleanup + [Documentation] Test Suit Cleanup: Deleting Descriptor, instance and vim + + Run Keyword If Any Tests Failed Delete NS ${ns_name} + Run Keyword If Any Tests Failed Delete NSD ${nsd_name} + Run Keyword If Any Tests Failed Delete VNFD ${vnfd_volumes_name} + Run Keyword If Any Tests Failed Delete VNFD ${vnfd_charm_name} diff --git a/robot-systest/testsuite/heal_02-scale_vdu_healing.robot b/robot-systest/testsuite/heal_02-scale_vdu_healing.robot new file mode 100644 index 0000000..4dd145b --- /dev/null +++ b/robot-systest/testsuite/heal_02-scale_vdu_healing.robot @@ -0,0 +1,176 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation [HEAL-02] Healing of scaled charm VDUs + +Library OperatingSystem +Library String +Library Collections +Library Process +Library SSHLibrary + +Resource %{ROBOT_DEVOPS_FOLDER}/lib/vnfd_lib.robot +Resource %{ROBOT_DEVOPS_FOLDER}/lib/vnf_lib.robot +Resource %{ROBOT_DEVOPS_FOLDER}/lib/nsd_lib.robot +Resource %{ROBOT_DEVOPS_FOLDER}/lib/ns_lib.robot +Resource %{ROBOT_DEVOPS_FOLDER}/lib/ssh_lib.robot +Resource %{ROBOT_DEVOPS_FOLDER}/lib/openstack_lib.robot + +Force Tags heal_02 cluster_main daily regression + + +Suite Teardown Run Keyword And Ignore Error Suite Cleanup + + +*** Variables *** +# NS and VNF descriptor package folder and ids +${vnfd_volumes_pkg} several_volumes_vnf +${vnfd_volumes_name} several_volumes-vnf +${vdu_volumes_name} several_volumes-VM +${vnf_several_index} several_volumes_vnf +${vnfd_charm_pkg} charm-packages/native_manual_scale_charm_vnf +${vnfd_charm_name} native_manual_scale_charm-vnf +${vdu_charm_name} mgmtVM +${vnf_charm_index} charm_vnf +${vnf_charm_scaling_group} manual-scaling_mgmtVM +${vnf_charm_cloudinit_file} /root/helloworld.txt +${vnf_charm_day1_file} /home/ubuntu/first-touch +${nsd_pkg} volumes_nativecharm_ns +${nsd_name} volumes_nativecharm-ns + +# NS instance name and configuration +${ns_name} heal_02 +${ns_config} {vld: [ {name: mgmtnet, vim-network-name: %{VIM_MGMT_NET}} ] } +${scale_wait_time} 4min + +# SSH keys and username to be used +${publickey} %{HOME}/.ssh/id_rsa.pub +${privatekey} %{HOME}/.ssh/id_rsa +${username} ubuntu +${password} ${EMPTY} + +${success_return_code} 0 + + +*** Test Cases *** +Create VNF Descriptors + [Tags] prepare + Create VNFD '%{PACKAGES_FOLDER}/${vnfd_charm_pkg}' + Create VNFD '%{PACKAGES_FOLDER}/${vnfd_volumes_pkg}' + + +Create NS Descriptor + [Tags] prepare + Create NSD '%{PACKAGES_FOLDER}/${nsd_pkg}' + + +Network Service Instance Test + [Tags] prepare + ${id}= Create Network Service ${nsd_name} %{VIM_TARGET} ${ns_name} ${ns_config} ${publickey} + Set Suite Variable ${ns_id} ${id} + + +Get NS Id + [Tags] verify cleanup + ${variables} Get Variables + IF not "\${ns_id}" in "${variables}" + ${id}= Get Ns Id ${ns_name} + Set Suite Variable ${ns_id} ${id} + END + + +Scale Out Charm VNF + [Tags] prepare + ${vnf_id}= Get Vnf Id ${ns_id} ${vnf_charm_index} + Set Suite Variable ${vnf_charm_id} ${vnf_id} + @{vdur_list}= Get Vnf Vdur Names ${vnf_charm_id} + ${vdurs}= Get Length ${vdur_list} + Set Suite Variable ${initial_vdur_count} ${vdurs} + Execute Manual VNF Scale ${ns_name} ${vnf_charm_index} ${vnf_charm_scaling_group} SCALE_OUT ${scale_wait_time} + @{vdur_list}= Get Vnf Vdur Names ${vnf_charm_id} + ${vdurs}= Get Length ${vdur_list} + Run Keyword Unless ${vdurs} == ${initial_vdur_count} + 1 Fail msg=There is no new VDU records in the VNF after Scale Out + + +Get Charm VNF Info + [Tags] verify + Variable Should Exist ${ns_id} msg=NS is not available + ${variables} Get Variables + IF not "\${vnf_charm_id}" in "${variables}" + ${vnf_id}= Get Vnf Id ${ns_id} ${vnf_charm_index} + Set Suite Variable ${vnf_charm_id} ${vnf_id} + END + ${id}= Get VNF VIM ID ${vnf_charm_id} + @{vdu_charm_ids}= Split String ${id} + Set Suite Variable @{vdu_charm_ids} @{vdu_charm_ids} + log ${vdu_charm_ids}[1] + @{charm_ip_list}= Get Vnf Vdur IPs ${vnf_charm_id} + Set Suite Variable @{charm_ip_list} @{charm_ip_list} + + +Stop Charm VDU + [Tags] verify + Variable Should Exist @{vdu_charm_ids} msg=VDU is not available + Stop Server ${vdu_charm_ids}[1] + Sleep 15 + + +Heal Charm VDU + [Tags] verify + Variable Should Exist ${vnf_charm_id} msg=VNF is not available + Heal Network Service ${ns_id} --vnf ${vnf_charm_id} --cause "Heal VM of charm_vnf" --vdu ${vdu_charm_name} --count-index 1 --run-day1 + + +Check VNF After Healing + [Tags] verify + Variable Should Exist ${vnf_charm_id} msg=VNF is not available + + @{ip_list}= Get Vnf Vdur IPs ${vnf_charm_id} + Should Be Equal ${ip_list} ${charm_ip_list} IP addresses have changed after healing + + ${id}= Get VNF VIM ID ${vnf_charm_id} + @{ids}= Split String ${id} + Should Be Equal ${vdu_charm_ids}[0] ${ids}[0] VDU[0] id has changed after healing + Should Not Be Equal ${vdu_charm_ids}[1] ${ids}[1] VDU[1] id has not changed after healing + Should Be Equal ${vdu_charm_ids}[2] ${ids}[2] VDU[2] id has changed after healing + + ${ip}= Get Vdu Attribute ${vnf_charm_id} ip-address 1 + ${stdout}= Execute Remote Command Check Rc Return Output ${ip} ${username} ${password} ${privatekey} sudo ls ${vnf_charm_cloudinit_file} + log ${stdout} + Check If remote File Exists ${ip} ${username} ${password} ${privatekey} ${vnf_charm_day1_file} + + +Delete NS Instance + [Tags] cleanup + Delete NS ${ns_name} + + +Delete NS Descriptor + [Tags] cleanup + Delete NSD ${nsd_name} + + +Delete VNF Descriptors + [Tags] cleanup + Delete VNFD ${vnfd_volumes_name} + Delete VNFD ${vnfd_charm_name} + + +*** Keywords *** +Suite Cleanup + [Documentation] Test Suit Cleanup: Deleting Descriptor, instance and vim + + Run Keyword If Any Tests Failed Delete NS ${ns_name} + Run Keyword If Any Tests Failed Delete NSD ${nsd_name} + Run Keyword If Any Tests Failed Delete VNFD ${vnfd_volumes_name} + Run Keyword If Any Tests Failed Delete VNFD ${vnfd_charm_name} diff --git a/robot-systest/testsuite/heal_03-multiple_healing.robot b/robot-systest/testsuite/heal_03-multiple_healing.robot new file mode 100644 index 0000000..3c2dd23 --- /dev/null +++ b/robot-systest/testsuite/heal_03-multiple_healing.robot @@ -0,0 +1,198 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation [HEAL-03] Healing of multiple VDUs + +Library OperatingSystem +Library String +Library Collections +Library Process +Library SSHLibrary + +Resource %{ROBOT_DEVOPS_FOLDER}/lib/vnfd_lib.robot +Resource %{ROBOT_DEVOPS_FOLDER}/lib/vnf_lib.robot +Resource %{ROBOT_DEVOPS_FOLDER}/lib/nsd_lib.robot +Resource %{ROBOT_DEVOPS_FOLDER}/lib/ns_lib.robot +Resource %{ROBOT_DEVOPS_FOLDER}/lib/ssh_lib.robot +Resource %{ROBOT_DEVOPS_FOLDER}/lib/openstack_lib.robot + +Force Tags heal_03 cluster_main daily regression + + +Suite Teardown Run Keyword And Ignore Error Suite Cleanup + + +*** Variables *** +# NS and VNF descriptor package folder and ids +${vnfd_volumes_pkg} several_volumes_vnf +${vnfd_volumes_name} several_volumes-vnf +${vdu_volumes_name} several_volumes-VM +${vnf_several_index} several_volumes_vnf +${vnfd_charm_pkg} charm-packages/native_manual_scale_charm_vnf +${vnfd_charm_name} native_manual_scale_charm-vnf +${vdu_charm_name} mgmtVM +${vnf_charm_index} charm_vnf +${vnf_charm_scaling_group} manual-scaling_mgmtVM +${vnf_charm_cloudinit_file} /root/helloworld.txt +${vnf_charm_day1_file} /home/ubuntu/first-touch +${nsd_pkg} volumes_nativecharm_ns +${nsd_name} volumes_nativecharm-ns + +# NS instance name and configuration +${ns_name} heal_03 +${ns_config} {vld: [ {name: mgmtnet, vim-network-name: %{VIM_MGMT_NET}} ] } +${scale_wait_time} 4min + +# SSH keys and username to be used +${publickey} %{HOME}/.ssh/id_rsa.pub +${privatekey} %{HOME}/.ssh/id_rsa +${username} ubuntu +${password} ${EMPTY} + +${success_return_code} 0 + + +*** Test Cases *** +Create VNF Descriptors + [Tags] prepare + Create VNFD '%{PACKAGES_FOLDER}/${vnfd_charm_pkg}' + Create VNFD '%{PACKAGES_FOLDER}/${vnfd_volumes_pkg}' + + +Create NS Descriptor + [Tags] prepare + Create NSD '%{PACKAGES_FOLDER}/${nsd_pkg}' + + +Network Service Instance Test + [Tags] prepare + ${id}= Create Network Service ${nsd_name} %{VIM_TARGET} ${ns_name} ${ns_config} ${publickey} + Set Suite Variable ${ns_id} ${id} + + +Get NS Id + [Tags] verify cleanup + ${variables} Get Variables + IF not "\${ns_id}" in "${variables}" + ${id}= Get Ns Id ${ns_name} + Set Suite Variable ${ns_id} ${id} + END + + +Scale Out Charm VNF + [Tags] prepare + ${vnf_id}= Get Vnf Id ${ns_id} ${vnf_charm_index} + Set Suite Variable ${vnf_charm_id} ${vnf_id} + @{vdur_list}= Get Vnf Vdur Names ${vnf_charm_id} + ${vdurs}= Get Length ${vdur_list} + Set Suite Variable ${initial_vdur_count} ${vdurs} + Execute Manual VNF Scale ${ns_name} ${vnf_charm_index} ${vnf_charm_scaling_group} SCALE_OUT ${scale_wait_time} + @{vdur_list}= Get Vnf Vdur Names ${vnf_charm_id} + ${vdurs}= Get Length ${vdur_list} + Run Keyword Unless ${vdurs} == ${initial_vdur_count} + 1 Fail msg=There is no new VDU records in the VNF after Scale Out + + +Get Charm VNF Info + [Tags] verify + Variable Should Exist ${ns_id} msg=NS is not available + ${variables} Get Variables + IF not "\${vnf_charm_id}" in "${variables}" + ${vnf_id}= Get Vnf Id ${ns_id} ${vnf_charm_index} + Set Suite Variable ${vnf_charm_id} ${vnf_id} + END + ${id}= Get VNF VIM ID ${vnf_charm_id} + @{vdu_charm_ids}= Split String ${id} + Set Suite Variable @{vdu_charm_ids} @{vdu_charm_ids} + log ${vdu_charm_ids}[1] + @{charm_ip_list}= Get Vnf Vdur IPs ${vnf_charm_id} + Set Suite Variable @{charm_ip_list} @{charm_ip_list} + + +Get Volume VNF Info + [Tags] verify + Variable Should Exist ${ns_id} msg=NS is not available + ${ip_addr}= Get Vnf Management Ip Address ${ns_id} ${vnf_several_index} + log ${ip_addr} + Set Suite Variable ${vnf_volumes_ip_addr} ${ip_addr} + + ${vnf_id}= Get Vnf Id ${ns_id} ${vnf_several_index} + Set Suite Variable ${vnf_volumes_id} ${vnf_id} + ${id}= Get VNF VIM ID ${vnf_id} + Set Suite Variable ${vdu_volumes_id} ${id} + log ${vdu_volumes_id} + + @{volumes_ip_list}= Get Vnf Vdur IPs ${vnf_volumes_id} + Set Suite Variable @{volumes_ip_list} @{volumes_ip_list} + log @{volumes_ip_list} + + +Stop VDUs + [Tags] verify + Variable Should Exist @{vdu_charm_ids} msg=Charm VDUs are not available + Variable Should Exist ${vdu_volumes_id} msg=Volume VDU is not available + Stop Server ${vdu_charm_ids}[1] + Stop Server ${vdu_charm_ids}[2] + Stop Server ${vdu_volumes_id} + Sleep 30 + + +Heal VDUs + [Tags] verify + Variable Should Exist ${vnf_charm_id} msg=Charm VNF is not available + Variable Should Exist ${vnf_volumes_id} msg=Volume VNF is not available + Heal Network Service ${ns_id} --vnf ${vnf_volumes_id} --cause "Heal myvdu1 of several_volumes_vnf" --vdu several_volumes-VM --vnf ${vnf_charm_id} --cause "Heal two VMs of native_manual_scale_charm_vnf" --vdu mgmtVM --count-index 1 --run-day1 --vdu mgmtVM --count-index 2 + + +Check VNFs After Healing + [Tags] verify + Variable Should Exist ${vnf_charm_id} msg=Charm VNF is not available + Variable Should Exist ${vnf_volumes_id} msg=Volume VNF is not available + ${id}= Get VNF VIM ID ${vnf_charm_id} + @{ids}= Split String ${id} + Should Be Equal ${vdu_charm_ids}[0] ${ids}[0] VDU[0] id has changed after healing + Should Not Be Equal ${vdu_charm_ids}[1] ${ids}[1] VDU[1] id has not changed after healing + Should Not Be Equal ${vdu_charm_ids}[2] ${ids}[2] VDU[2] id has not changed after healing + + ${id}= Get VNF VIM ID ${vnf_volumes_id} + Should Not Be Equal ${id} ${vdu_volumes_id} VDU id has not changed after healing + + ${ip}= Get Vdu Attribute ${vnf_charm_id} ip-address 2 + ${stdout}= Execute Remote Command Check Rc Return Output ${ip} ${username} ${password} ${privatekey} sudo ls ${vnf_charm_cloudinit_file} + log ${stdout} + Check If remote File Exists ${ip} ${username} ${password} ${privatekey} ${vnf_charm_day1_file} + + +Delete NS Instance + [Tags] cleanup + Delete NS ${ns_name} + + +Delete NS Descriptor + [Tags] cleanup + Delete NSD ${nsd_name} + + +Delete VNF Descriptors + [Tags] cleanup + Delete VNFD ${vnfd_volumes_name} + Delete VNFD ${vnfd_charm_name} + + +*** Keywords *** +Suite Cleanup + [Documentation] Test Suit Cleanup: Deleting Descriptor, instance and vim + + Run Keyword If Any Tests Failed Delete NS ${ns_name} + Run Keyword If Any Tests Failed Delete NSD ${nsd_name} + Run Keyword If Any Tests Failed Delete VNFD ${vnfd_volumes_name} + Run Keyword If Any Tests Failed Delete VNFD ${vnfd_charm_name} -- 2.17.1