2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
16 Documentation [HEAL-02] Healing of scaled VDUs
18 Library OperatingSystem
24 Resource ../lib/vnfd_lib.resource
25 Resource ../lib/vnf_lib.resource
26 Resource ../lib/nsd_lib.resource
27 Resource ../lib/ns_lib.resource
28 Resource ../lib/ns_operation_lib.resource
29 Resource ../lib/ssh_lib.resource
30 Resource ../lib/openstack_lib.resource
32 Test Tags heal_02 cluster_heal daily regression
34 Suite Teardown Run Keyword And Ignore Error Suite Cleanup
38 # NS and VNF descriptor package folder and ids
39 ${VNFD_VOLUMES_PKG} several_volumes_vnf
40 ${VNFD_VOLUMES_NAME} several_volumes-vnf
41 ${VDU_VOLUMES_NAME} several_volumes-VM
42 ${VNF_SEVERAL_INDEX} several_volumes_vnf
43 ${VNFD_MANUALSCALE_PKG} manual_scale_vnf
44 ${VNFD_MANUALSCALE_NAME} manual_scale-vnf
45 ${VDU_MANUALSCALE_NAME} mgmtVM
46 ${VNF_MANUALSCALE_INDEX} manual_scale_vnf
47 ${VNF_MANUALSCALE_SCALING_GROUP} manual-scaling_mgmtVM
48 ${VNF_MANUALSCALE_CLOUDINIT_FILE} /root/helloworld.txt
49 ${VNF_MANUALSCALE_DAY1_FILE} /home/ubuntu/first-touch
50 ${NSD_PKG} volumes_healing_ns
51 ${NSD_NAME} volumes_healing-ns
53 # NS instance name and configuration
54 ${FLAVOR_NAME_PREFIX} osm.heal02
57 ${SCALE_WAIT_TIME} 5min
59 # SSH keys and username to be used
60 ${PUBLICKEY} %{HOME}/.ssh/id_rsa.pub
61 ${PRIVATEKEY} %{HOME}/.ssh/id_rsa
65 ${SUCCESS_RETURN_CODE} 0
68 @{VIM_VOLUMES} @{EMPTY}
70 # Variables to control time for healing and polling
71 ${VNF_MAX_TIME_TO_BE_READY} 120sec
76 Create VNF Descriptors
77 [Documentation] Upload VNF packages for the testsuite.
78 Create VNFD '%{PACKAGES_FOLDER}/${VNFD_MANUALSCALE_PKG}'
79 Create VNFD '%{PACKAGES_FOLDER}/${VNFD_VOLUMES_PKG}'
82 [Documentation] Upload NS package for the testsuite.
83 Create NSD '%{PACKAGES_FOLDER}/${NSD_PKG}'
86 [Documentation] Create a flavor that will be used at NS instantiation time and save it as FLAVOR_ID.
87 ${rand}= Generate Random String 6 [NUMBERS]
88 ${flavor_name}= Catenate SEPARATOR=_ ${FLAVOR_NAME_PREFIX} ${rand}
89 ${id}= Create Flavor ${flavor_name} 1 1024 10
90 Set Suite Variable ${FLAVOR_ID} ${id}
92 Network Service Instance Test
93 [Documentation] Instantiate NS for the testsuite using the previously created flavor.
94 ${ns_config}= Set Variable {vld: [ {name: mgmtnet, vim-network-name: %{VIM_MGMT_NET}}],vnf: [ {member-vnf-index: manual_scale_vnf, vdu: [{ id: mgmtVM, vim-flavor-id: ${FLAVOR_ID}}]}] }
95 ${id}= Create Network Service ${NSD_NAME} %{VIM_TARGET} ${NS_NAME} ${ns_config} ${PUBLICKEY} ${NS_TIMEOUT}
96 Set Suite Variable ${NS_ID} ${id}
99 [Documentation] Get NS identifier.
101 ${variables}= Get Variables
102 IF not "\${ns_id}" in "${variables}"
103 ${id}= Get Ns Id ${NS_NAME}
104 Set Suite Variable ${NS_ID} ${id}
107 Scale Out Manual Scale VNF
108 [Documentation] Perform a manual scale-out operation of the manual-scale VNF.
109 ${vnf_id}= Get Vnf Id ${NS_ID} ${VNF_MANUALSCALE_INDEX}
110 Set Suite Variable ${VNF_MANUALSCALE_ID} ${vnf_id}
111 @{vdur_list}= Get Vnf Vdur Names ${VNF_MANUALSCALE_ID}
112 ${vdurs}= Get Length ${vdur_list}
113 Set Suite Variable ${INITIAL_VDUR_COUNT} ${vdurs}
114 Execute Manual VNF Scale ${NS_NAME} ${VNF_MANUALSCALE_INDEX} ${VNF_MANUALSCALE_SCALING_GROUP} SCALE_OUT ${SCALE_WAIT_TIME}
115 @{vdur_list}= Get Vnf Vdur Names ${VNF_MANUALSCALE_ID}
116 ${vdurs}= Get Length ${vdur_list}
117 IF ${vdurs} != ${INITIAL_VDUR_COUNT} + 1 Fail msg=There is no new VDU records in the VNF after Scale Out
120 [Documentation] Retrieve all VMs and volumes from the NS and stores them in VIM_VDUS and VIM_VOLUMES lists.
121 Variable Should Exist ${NS_ID} msg=NS is not available
122 @{vnf_id_list}= Get Ns Vnf List ${NS_ID}
124 FOR ${vnf_id} IN @{vnf_id_list}
126 ${id}= Get VNF VIM ID ${vnf_id}
127 @{vdu_ids}= Split String ${id}
128 Append To List ${VIM_VDUS} @{vdu_ids}
130 FOR ${vdu_id} IN @{VIM_VDUS}
131 ${volumes_attached}= Get Server Property ${vdu_id} volumes_attached
132 ${match}= Get Regexp Matches ${volumes_attached} '([0-9a-f\-]+)' 1
133 IF ${match} != @{EMPTY}
134 IF not "${match}[0]" in "@{VIM_VOLUMES}"
135 Append To List ${VIM_VOLUMES} ${match}[0]
140 Log Many @{VIM_VOLUMES}
142 Get Manual Scale VNF Info
143 [Documentation] Get VDU ID and IP addresses of the manual scale VNF and stores them in VDU_MANUALSCALE_IDS and MANUALSCALE_IP_LIST.
144 Variable Should Exist ${NS_ID} msg=NS is not available
145 ${variables}= Get Variables
146 IF not "\${VNF_MANUALSCALE_ID}" in "${variables}"
147 ${vnf_id}= Get Vnf Id ${NS_ID} ${VNF_MANUALSCALE_INDEX}
148 Set Suite Variable ${VNF_MANUALSCALE_ID} ${vnf_id}
150 ${id}= Get VNF VIM ID ${VNF_MANUALSCALE_ID}
151 @{vdu_manualscale_ids}= Split String ${id}
152 Set Suite Variable @{VDU_MANUALSCALE_IDS} @{vdu_manualscale_ids}
153 Log ${VDU_MANUALSCALE_IDS}[1]
154 @{manualscale_ip_list}= Get Vnf Vdur IPs ${VNF_MANUALSCALE_ID}
155 Set Suite Variable @{MANUALSCALE_IP_LIST} @{manualscale_ip_list}
156 ${ip}= Get Vdu Attribute ${VNF_MANUALSCALE_ID} ip-address 1
157 Set Suite Variable ${HEALED_VDU_MGMT_IP} ${ip}
159 Check Day0 And Day1 In VDU Before Halting VM
160 [Documentation] Check that the VDU is accessible via SSH in its mgmt IP address.
161 ... It also checks if day-0 worked and a remote file has been created in the VDU.
162 ... It also checks if day-1 worked and a remote file has been created in the VDU.
163 ${ip}= Get Vdu Attribute ${VNF_MANUALSCALE_ID} ip-address 1
164 Variable Should Exist ${HEALED_VDU_MGMT_IP} msg=IP address of the healed VDU is not available
165 Wait Until Keyword Succeeds ${VNF_MAX_TIME_TO_BE_READY} ${VNF_POL_TIME} Test SSH Connection ${HEALED_VDU_MGMT_IP} ${USERNAME} ${PASSWORD} ${PRIVATEKEY}
166 ${stdout}= Execute Remote Command Check Rc Return Output ${ip} ${USERNAME} ${PASSWORD} ${PRIVATEKEY} sudo ls ${VNF_MANUALSCALE_CLOUDINIT_FILE}
168 # Check If Remote File Exists ${HEALED_VDU_MGMT_IP} ${USERNAME} ${PASSWORD} ${PRIVATEKEY} ${VNF_MANUALSCALE_DAY1_FILE}
170 Halt Manual Scale VDU
171 [Documentation] Halt one of the VM of the Manual Scale VNF.
172 Variable Should Exist @{VDU_MANUALSCALE_IDS} msg=VDU is not available
173 Halt Server ${VDU_MANUALSCALE_IDS}[1]
176 Heal Manual Scale VDU
177 [Documentation] Heal manually via OSM commands all stopped VMs . They should be started again.
178 Variable Should Exist ${VNF_MANUALSCALE_ID} msg=VNF is not available
179 Heal Network Service ${NS_ID} --vnf ${VNF_MANUALSCALE_ID} --cause "Heal VM of manual_scale_vnf" --vdu ${VDU_MANUALSCALE_NAME} --count-index 1 --run-day1
181 Check VNF After Healing
182 [Documentation] Check that the IDs of the VM and volumes have not changed after healing.
183 Variable Should Exist ${VNF_MANUALSCALE_ID} msg=VNF is not available
184 @{ip_list}= Get Vnf Vdur IPs ${VNF_MANUALSCALE_ID}
185 Should Be Equal ${ip_list} ${MANUALSCALE_IP_LIST} IP addresses have changed after healing
186 ${id}= Get VNF VIM ID ${VNF_MANUALSCALE_ID}
187 @{ids}= Split String ${id}
188 Should Be Equal ${VDU_MANUALSCALE_IDS}[0] ${ids}[0] VDU[0] id has changed after healing
189 Should Not Be Equal ${VDU_MANUALSCALE_IDS}[1] ${ids}[1] VDU[1] id has not changed after healing
190 Should Be Equal ${VDU_MANUALSCALE_IDS}[2] ${ids}[2] VDU[2] id has changed after healing
191 ${vim_info}= Get Vdu Attribute ${VNF_MANUALSCALE_ID} vim_info 1
192 Should Contain ${vim_info} id: ${FLAVOR_ID} msg=Flavor ID is incorrect
193 ${ip}= Get Vdu Attribute ${VNF_MANUALSCALE_ID} ip-address 1
194 Should Be Equal ${HEALED_VDU_MGMT_IP} ${ip}
196 Check Day0 And Day1 In VDU After Healing
197 [Documentation] Check that the healed VDU is accessible via SSH in its mgmt IP address.
198 ... It also checks if day-0 worked after healing and a remote file has been created in the VDU.
199 ... It also checks if day-1 worked after healing and a remote file has been created in the VDU.
200 Variable Should Exist ${HEALED_VDU_MGMT_IP} msg=IP address of the healed VDU is not available
201 Wait Until Keyword Succeeds ${VNF_MAX_TIME_TO_BE_READY} ${VNF_POL_TIME} Test SSH Connection ${HEALED_VDU_MGMT_IP} ${USERNAME} ${PASSWORD} ${PRIVATEKEY}
202 ${stdout}= Execute Remote Command Check Rc Return Output ${HEALED_VDU_MGMT_IP} ${USERNAME} ${PASSWORD} ${PRIVATEKEY} sudo ls ${VNF_MANUALSCALE_CLOUDINIT_FILE}
204 # Check If Remote File Exists ${HEALED_VDU_MGMT_IP} ${USERNAME} ${PASSWORD} ${PRIVATEKEY} ${VNF_MANUALSCALE_DAY1_FILE}
207 [Documentation] Retrieve all VMs and volumes from the NS and stores them in VIM_VDUS and VIM_VOLUMES lists.
208 ... This is done again to guarantee that all objects are cleaned in the VIM in case the heal operation
209 ... added new objects.
210 Variable Should Exist ${NS_ID} msg=NS is not available
211 @{vdu_updated}= Create List
212 @{vnf_id_list}= Get Ns Vnf List ${NS_ID}
213 FOR ${vnf_id} IN @{vnf_id_list}
214 ${id}= Get VNF VIM ID ${vnf_id}
215 @{vdu_ids}= Split String ${id}
216 Append To List ${vdu_updated} @{vdu_ids}
217 FOR ${id} IN @{vdu_ids}
218 IF not "${id}" in "@{VIM_VDUS}"
219 Append To List ${VIM_VDUS} ${id}
223 FOR ${vdu_id} IN @{vdu_updated}
224 ${volumes_attached}= Get Server Property ${vdu_id} volumes_attached
225 ${match}= Get Regexp Matches ${volumes_attached} '([0-9a-f\-]+)' 1
226 IF ${match} != @{EMPTY}
227 IF not "${match}[0]" in "@{VIM_VOLUMES}"
228 Append To List ${VIM_VOLUMES} ${match}[0]
233 Log Many @{VIM_VOLUMES}
236 [Documentation] Delete NS instance.
241 [Documentation] Delete NS package from OSM.
243 Delete NSD ${NSD_NAME}
245 Delete VNF Descriptors
246 [Documentation] Delete VNF packages from OSM.
248 Delete VNFD ${VNFD_VOLUMES_NAME}
249 Delete VNFD ${VNFD_MANUALSCALE_NAME}
251 Delete Remaining Objects In VIM
252 [Documentation] Delete any remaining objects (volumes, VMs, etc.) in the VIM.
254 Delete Objects In VIM
259 [Documentation] Test Suit Cleanup: Deleting Descriptor, instance and vim
260 Run Keyword If Any Tests Failed Delete NS ${NS_NAME}
261 Run Keyword If Any Tests Failed Delete NSD ${NSD_NAME}
262 Run Keyword If Any Tests Failed Delete VNFD ${VNFD_VOLUMES_NAME}
263 Run Keyword If Any Tests Failed Delete VNFD ${VNFD_MANUALSCALE_NAME}
264 Run Keyword If Any Tests Failed Delete Objects In VIM
266 Delete Objects In VIM
267 [Documentation] Clean up remaining VMs and volumes directly from the VIM.
268 Delete Flavor ${FLAVOR_ID}
269 ${error}= Set Variable 0
270 FOR ${vol_id} IN @{VIM_VOLUMES}
271 Log Checking if volume ${vol_id} is still in VIM
272 ${exists}= Check If Volume Exists ${vol_id}
274 ${error}= Set Variable 1
275 Log Deleting volume ${vol_id}
276 Run Keyword And Ignore Error Delete Volume ${vol_id}
279 FOR ${vdu_id} IN @{VIM_VDUS}
280 Log Checking if server ${vdu_id} is still in VIM
281 ${status}= Run Keyword And Ignore Error Get Server Property ${vdu_id} id
283 IF '${status}[0]' == 'PASS'
284 ${error}= Set Variable 1
285 Log Deleting server ${vdu_id}
286 Run Keyword And Ignore Error Delete Server ${vdu_id}
290 Fail Some objects created by test were not deleted in VIM