2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
16 Documentation [HEAL-03] Healing of multiple VDUs
18 Library OperatingSystem
24 Resource ../lib/vnfd_lib.resource
25 Resource ../lib/vnf_lib.resource
26 Resource ../lib/nsd_lib.resource
27 Resource ../lib/ns_lib.resource
28 Resource ../lib/ns_operation_lib.resource
29 Resource ../lib/ssh_lib.resource
30 Resource ../lib/openstack_lib.resource
32 Test Tags heal_03 cluster_heal daily regression
34 Suite Teardown Run Keyword And Ignore Error Suite Cleanup
38 # NS and VNF descriptor package folder and ids
39 ${VNFD_VOLUMES_PKG} several_volumes_vnf
40 ${VNFD_VOLUMES_NAME} several_volumes-vnf
41 ${VDU_VOLUMES_NAME} several_volumes-VM
42 ${VNF_SEVERAL_INDEX} several_volumes_vnf
43 ${VNFD_MANUALSCALE_PKG} manual_scale_vnf
44 ${VNFD_MANUALSCALE_NAME} manual_scale-vnf
45 ${VDU_MANUALSCALE_NAME} mgmtVM
46 ${VNF_MANUALSCALE_INDEX} manual_scale_vnf
47 ${VNF_MANUALSCALE_SCALING_GROUP} manual-scaling_mgmtVM
48 ${VNF_MANUALSCALE_CLOUDINIT_FILE} /root/helloworld.txt
49 ${VNF_MANUALSCALE_DAY1_FILE} /home/ubuntu/first-touch
50 ${NSD_PKG} volumes_healing_ns
51 ${NSD_NAME} volumes_healing-ns
53 # NS instance name and configuration
55 ${NS_CONFIG} {vld: [ {name: mgmtnet, vim-network-name: %{VIM_MGMT_NET}} ] }
57 ${SCALE_WAIT_TIME} 5min
59 # SSH keys and username to be used
60 ${PUBLICKEY} %{HOME}/.ssh/id_rsa.pub
61 ${PRIVATEKEY} %{HOME}/.ssh/id_rsa
65 ${SUCCESS_RETURN_CODE} 0
68 @{VIM_VOLUMES} @{EMPTY}
70 # Variables to control time for healing and polling
71 ${VNF_MAX_TIME_TO_BE_READY} 120sec
76 Create VNF Descriptors
77 [Documentation] Upload VNF packages for the testsuite.
78 Create VNFD '%{PACKAGES_FOLDER}/${VNFD_MANUALSCALE_PKG}'
79 Create VNFD '%{PACKAGES_FOLDER}/${VNFD_VOLUMES_PKG}'
82 [Documentation] Upload NS package for the testsuite.
83 Create NSD '%{PACKAGES_FOLDER}/${NSD_PKG}'
85 Network Service Instance Test
86 [Documentation] Instantiate NS for the testsuite.
87 ${id}= Create Network Service ${NSD_NAME} %{VIM_TARGET} ${NS_NAME} ${NS_CONFIG} ${PUBLICKEY} ${NS_TIMEOUT}
88 Set Suite Variable ${NS_ID} ${id}
91 [Documentation] Get NS identifier and stores as suite variable to be used later on.
93 ${variables}= Get Variables
94 IF not "\${ns_id}" in "${variables}"
95 ${id}= Get Ns Id ${NS_NAME}
96 Set Suite Variable ${NS_ID} ${id}
99 Scale Out Manual Scale VNF
100 [Documentation] Perform a manual scale-out operation of the manual scale VNF.
101 ${vnf_id}= Get Vnf Id ${NS_ID} ${VNF_MANUALSCALE_INDEX}
102 Set Suite Variable ${VNF_MANUALSCALE_ID} ${vnf_id}
103 @{vdur_list}= Get Vnf Vdur Names ${VNF_MANUALSCALE_ID}
104 ${vdurs}= Get Length ${vdur_list}
105 Set Suite Variable ${INITIAL_VDUR_COUNT} ${vdurs}
106 Execute Manual VNF Scale ${NS_NAME} ${VNF_MANUALSCALE_INDEX} ${VNF_MANUALSCALE_SCALING_GROUP} SCALE_OUT ${SCALE_WAIT_TIME}
107 @{vdur_list}= Get Vnf Vdur Names ${VNF_MANUALSCALE_ID}
108 ${vdurs}= Get Length ${vdur_list}
109 IF ${vdurs} != ${INITIAL_VDUR_COUNT} + 1 Fail msg=There is no new VDU records in the VNF after Scale Out
112 [Documentation] Retrieve all VMs and volumes from the NS and stores them in VIM_VDUS and VIM_VOLUMES lists.
113 Variable Should Exist ${NS_ID} msg=NS is not available
114 @{vnf_id_list}= Get Ns Vnf List ${NS_ID}
116 FOR ${vnf_id} IN @{vnf_id_list}
118 ${id}= Get VNF VIM ID ${vnf_id}
119 @{vdu_ids}= Split String ${id}
120 Append To List ${VIM_VDUS} @{vdu_ids}
122 FOR ${vdu_id} IN @{VIM_VDUS}
123 ${volumes_attached}= Get Server Property ${vdu_id} volumes_attached
124 ${match}= Get Regexp Matches ${volumes_attached} '([0-9a-f\-]+)' 1
125 IF ${match} != @{EMPTY}
126 IF not "${match}[0]" in "@{VIM_VOLUMES}"
127 Append To List ${VIM_VOLUMES} ${match}[0]
132 Log Many @{VIM_VOLUMES}
134 Get Manual Scale VNF Info
135 [Documentation] Get VDU ID and IP addresses of the manual scale VNF and stores them in VDU_MANUALSCALE_IDS and MANUALSCALE_IP_LIST.
136 Variable Should Exist ${NS_ID} msg=NS is not available
137 ${variables}= Get Variables
138 IF not "\${VNF_MANUALSCALE_ID}" in "${variables}"
139 ${vnf_id}= Get Vnf Id ${NS_ID} ${VNF_MANUALSCALE_INDEX}
140 Set Suite Variable ${VNF_MANUALSCALE_ID} ${vnf_id}
142 ${id}= Get VNF VIM ID ${VNF_MANUALSCALE_ID}
143 @{vdu_manualscale_ids}= Split String ${id}
144 Set Suite Variable @{VDU_MANUALSCALE_IDS} @{vdu_manualscale_ids}
145 Log ${VDU_MANUALSCALE_IDS}[1]
146 @{manualscale_ip_list}= Get Vnf Vdur IPs ${VNF_MANUALSCALE_ID}
147 Set Suite Variable @{MANUALSCALE_IP_LIST} @{manualscale_ip_list}
148 ${ip}= Get Vdu Attribute ${VNF_MANUALSCALE_ID} ip-address 1
149 Set Suite Variable ${HEALED_VDU_MGMT_IP} ${ip}
152 [Documentation] Get info about volumes and VM of the VNF with several volumes and stores them in suite variables.
153 Variable Should Exist ${NS_ID} msg=NS is not available
154 ${ip_addr}= Get Vnf Management Ip Address ${NS_ID} ${VNF_SEVERAL_INDEX}
156 Set Suite Variable ${VNF_VOLUMES_IP_ADDR} ${ip_addr}
157 ${vnf_id}= Get Vnf Id ${NS_ID} ${VNF_SEVERAL_INDEX}
158 Set Suite Variable ${VNF_VOLUMES_ID} ${vnf_id}
159 ${id}= Get VNF VIM ID ${vnf_id}
160 Set Suite Variable ${VDU_VOLUMES_ID} ${id}
161 Log ${VDU_VOLUMES_ID}
162 @{volumes_ip_list}= Get Vnf Vdur IPs ${VNF_VOLUMES_ID}
163 Set Suite Variable @{VOLUMES_IP_LIST} @{volumes_ip_list}
164 Log @{VOLUMES_IP_LIST}
166 Check Day0 And Day1 In VDU Before Stopping VM
167 [Documentation] Check that the VDU is accessible via SSH in its mgmt IP address.
168 ... It also checks if day-0 worked and a remote file has been created in the VDU.
169 ... It also checks if day-1 worked and a remote file has been created in the VDU.
170 ${ip}= Get Vdu Attribute ${VNF_MANUALSCALE_ID} ip-address 1
171 Variable Should Exist ${HEALED_VDU_MGMT_IP} msg=IP address of the healed VDU is not available
172 Wait Until Keyword Succeeds ${VNF_MAX_TIME_TO_BE_READY} ${VNF_POL_TIME} Test SSH Connection ${HEALED_VDU_MGMT_IP} ${USERNAME} ${PASSWORD} ${PRIVATEKEY}
173 ${stdout}= Execute Remote Command Check Rc Return Output ${ip} ${USERNAME} ${PASSWORD} ${PRIVATEKEY} sudo ls ${VNF_MANUALSCALE_CLOUDINIT_FILE}
175 # Check If Remote File Exists ${HEALED_VDU_MGMT_IP} ${USERNAME} ${PASSWORD} ${PRIVATEKEY} ${VNF_MANUALSCALE_DAY1_FILE}
178 [Documentation] Stop all VMs.
179 Variable Should Exist @{VDU_MANUALSCALE_IDS} msg=Manual Scale VDUs are not available
180 Variable Should Exist ${VDU_VOLUMES_ID} msg=Volume VDU is not available
181 Stop Server ${VDU_MANUALSCALE_IDS}[1]
182 Stop Server ${VDU_MANUALSCALE_IDS}[2]
183 Stop Server ${VDU_VOLUMES_ID}
187 [Documentation] Heal manually via OSM commands all stopped VMs . They should be started again.
188 Variable Should Exist ${VNF_MANUALSCALE_ID} msg=Manual Scale VNF is not available
189 Variable Should Exist ${VNF_VOLUMES_ID} msg=Volume VNF is not available
190 Heal Network Service ${NS_ID} --vnf ${VNF_VOLUMES_ID} --cause "Heal myvdu1 of several_volumes_vnf" --vdu several_volumes-VM --vnf ${VNF_MANUALSCALE_ID} --cause "Heal two VMs of manual_scale_vnf" --vdu mgmtVM --count-index 1 --run-day1 --vdu mgmtVM --count-index 2
192 Check VNFs After Healing
193 [Documentation] Check that the IDs of the VM and volumes have not changed after healing.
194 Variable Should Exist ${VNF_MANUALSCALE_ID} msg=Manual scale VNF is not available
195 Variable Should Exist ${VNF_VOLUMES_ID} msg=Volume VNF is not available
196 ${id}= Get VNF VIM ID ${VNF_MANUALSCALE_ID}
197 @{ids}= Split String ${id}
198 Should Be Equal ${VDU_MANUALSCALE_IDS}[0] ${ids}[0] VDU[0] id has changed after healing
199 Should Not Be Equal ${VDU_MANUALSCALE_IDS}[1] ${ids}[1] VDU[1] id has not changed after healing
200 Should Not Be Equal ${VDU_MANUALSCALE_IDS}[2] ${ids}[2] VDU[2] id has not changed after healing
201 ${id}= Get VNF VIM ID ${VNF_VOLUMES_ID}
202 Should Not Be Equal ${id} ${VDU_VOLUMES_ID} VDU id has not changed after healing
203 ${ip}= Get Vdu Attribute ${VNF_MANUALSCALE_ID} ip-address 1
204 Should Be Equal ${HEALED_VDU_MGMT_IP} ${ip}
206 Check Day0 And Day1 In VDU After Healing
207 [Documentation] Check that the healed VDU is accessible via SSH in its mgmt IP address.
208 ... It also checks if day-0 worked after healing and a remote file has been created in the VDU.
209 ... It also checks if day-1 worked after healing and a remote file has been created in the VDU.
210 Variable Should Exist ${HEALED_VDU_MGMT_IP} msg=IP address of the healed VDU is not available
211 Wait Until Keyword Succeeds ${VNF_MAX_TIME_TO_BE_READY} ${VNF_POL_TIME} Test SSH Connection ${HEALED_VDU_MGMT_IP} ${USERNAME} ${PASSWORD} ${PRIVATEKEY}
212 ${stdout}= Execute Remote Command Check Rc Return Output ${HEALED_VDU_MGMT_IP} ${USERNAME} ${PASSWORD} ${PRIVATEKEY} sudo ls ${VNF_MANUALSCALE_CLOUDINIT_FILE}
214 # Check If Remote File Exists ${HEALED_VDU_MGMT_IP} ${USERNAME} ${PASSWORD} ${PRIVATEKEY} ${VNF_MANUALSCALE_DAY1_FILE}
217 [Documentation] Retrieve all VMs and volumes from the NS and stores them in VIM_VDUS and VIM_VOLUMES lists.
218 ... This is done again to guarantee that all objects are cleaned in the VIM in case the heal operation
219 ... added new objects.
220 Variable Should Exist ${NS_ID} msg=NS is not available
221 @{vdu_updated}= Create List
222 @{vnf_id_list}= Get Ns Vnf List ${NS_ID}
223 FOR ${vnf_id} IN @{vnf_id_list}
224 ${id}= Get VNF VIM ID ${vnf_id}
225 @{vdu_ids}= Split String ${id}
226 Append To List ${vdu_updated} @{vdu_ids}
227 FOR ${id} IN @{vdu_ids}
228 IF not "${id}" in "@{VIM_VDUS}"
229 Append To List ${VIM_VDUS} ${id}
233 FOR ${vdu_id} IN @{vdu_updated}
234 ${volumes_attached}= Get Server Property ${vdu_id} volumes_attached
235 ${match}= Get Regexp Matches ${volumes_attached} '([0-9a-f\-]+)' 1
236 IF ${match} != @{EMPTY}
237 IF not "${match}[0]" in "@{VIM_VOLUMES}"
238 Append To List ${VIM_VOLUMES} ${match}[0]
243 Log Many @{VIM_VOLUMES}
246 [Documentation] Delete NS instance.
251 [Documentation] Delete NS package from OSM.
253 Delete NSD ${NSD_NAME}
255 Delete VNF Descriptors
256 [Documentation] Delete VNF packages from OSM.
258 Delete VNFD ${VNFD_VOLUMES_NAME}
259 Delete VNFD ${VNFD_MANUALSCALE_NAME}
261 Delete Objects In VIM
262 [Documentation] Delete any remaining objects (volumes, VMs, etc.) in the VIM.
264 ${error}= Set Variable 0
265 FOR ${vol_id} IN @{VIM_VOLUMES}
266 Log Checking if volume ${vol_id} is still in VIM
267 ${exists}= Check If Volume Exists ${vol_id}
269 ${error}= Set Variable 1
270 Log Deleting volume ${vol_id}
271 Run Keyword And Ignore Error Delete Volume ${vol_id}
274 FOR ${vdu_id} IN @{VIM_VDUS}
275 Log Checking if server ${vdu_id} is still in VIM
276 ${status}= Run Keyword And Ignore Error Get Server Property ${vdu_id} id
278 IF '${status}[0]' == 'PASS'
279 ${error}= Set Variable 1
280 Log Deleting server ${vdu_id}
281 Run Keyword And Ignore Error Delete Server ${vdu_id}
285 Fail Some objects created by test were not deleted in VIM
291 [Documentation] Test Suit Cleanup: Deleting Descriptor, instance and vim
292 Run Keyword If Any Tests Failed Delete NS ${NS_NAME}
293 Run Keyword If Any Tests Failed Delete NSD ${NSD_NAME}
294 Run Keyword If Any Tests Failed Delete VNFD ${VNFD_VOLUMES_NAME}
295 Run Keyword If Any Tests Failed Delete VNFD ${VNFD_MANUALSCALE_NAME}