Feature 10909: Robot tests for healing

Change-Id: I86bab7a72baf6f4accec74e2d54a7b4dfdbe6c38
Signed-off-by: aguilard <e.dah.tid@telefonica.com>
diff --git a/robot-systest/lib/ns_lib.robot b/robot-systest/lib/ns_lib.robot
index 7c2b794..3cd863e 100644
--- a/robot-systest/lib/ns_lib.robot
+++ b/robot-systest/lib/ns_lib.robot
@@ -23,6 +23,7 @@
 ${ns_action_max_wait_time}   1min
 ${ns_action_pol_time}   15sec
 ${vnf_scale_pol_time}   15sec
+${healing_pol_time}   15sec
 ${vim_timeout_multiplier}   %{OSM_VIM_MULTIPLIER_TIMEOUT=1.0}
 
 
@@ -196,7 +197,8 @@
     ${osm_ns_action_command}=  Catenate  ${osm_ns_action_command}  ${ns_name}
     ${rc}  ${stdout}=  Run and Return RC and Output  ${osm_ns_action_command}
     Should Be Equal As Integers  ${rc}  ${success_return_code}  msg=${stdout}  values=False
-    Wait Until Keyword Succeeds  ${ns_action_max_wait_time}  ${ns_action_pol_time}  Check For NS Operation Completed  ${stdout}
+    Wait Until Keyword Succeeds  ${ns_action_max_wait_time}  ${ns_action_pol_time}  Check For NS Operation Ended  ${stdout}
+    Check For NS Operation Completed  ${stdout}
     [Return]  ${stdout}
 
 
@@ -221,7 +223,8 @@
     ${osm_ns_action_command}=  Catenate  ${osm_ns_action_command}  ${ns_name}
     ${rc}  ${stdout}=  Run and Return RC and Output  ${osm_ns_action_command}
     Should Be Equal As Integers  ${rc}  ${success_return_code}  msg=${stdout}  values=False
-    Wait Until Keyword Succeeds  ${ns_action_max_wait_time}  ${ns_action_pol_time}  Check For NS Operation Completed  ${stdout}
+    Wait Until Keyword Succeeds  ${ns_action_max_wait_time}  ${ns_action_pol_time}  Check For NS Operation Ended  ${stdout}
+    Check For NS Operation Completed  ${stdout}
     [Return]  ${stdout}
 
 
@@ -241,13 +244,32 @@
     ${osm_vnf_scale_command}=  Catenate  ${osm_vnf_scale_command}  ${ns_name}  ${vnf_member_index}
     ${rc}  ${stdout}=  Run and Return RC and Output  ${osm_vnf_scale_command}
     Should Be Equal As Integers  ${rc}  ${success_return_code}  msg=${stdout}  values=False
-    Wait Until Keyword Succeeds  ${vnf_scale_max_wait_time}  ${vnf_scale_pol_time}  Check For NS Operation Completed  ${stdout}
+    Wait Until Keyword Succeeds  ${vnf_scale_max_wait_time}  ${vnf_scale_pol_time}  Check For NS Operation Ended  ${stdout}
+    Check For NS Operation Completed  ${stdout}
+    [Return]  ${stdout}
+
+
+Heal Network Service
+    [Documentation]     Execute healing operation over one NS.
+        ...             Return the ID of the operation associated to the executed healing action.
+
+    [Arguments]   ${ns_name}   ${params}   ${healing_max_wait_time}=10m
+
+    Should Not Be Empty   ${ns_name}
+    Should Not Be Empty   ${params}
+    ${healing_max_wait_time}=   Convert Time   ${healing_max_wait_time}   result_format=number
+    ${healing_max_wait_time}=   Evaluate   ${healing_max_wait_time} * ${vim_timeout_multiplier}
+    ${rc}  ${stdout}=  Run and Return RC and Output   osm ns-heal ${ns_name} ${params}
+    Should Be Equal As Integers  ${rc}  ${success_return_code}  msg=${stdout}  values=False
+    Wait Until Keyword Succeeds  ${healing_max_wait_time}  ${healing_pol_time}  Check For NS Operation Ended  ${stdout}
+    Check For NS Operation Completed  ${stdout}
     [Return]  ${stdout}
 
 
 Get Operations List
     [Arguments]  ${ns_name}
 
+    Should Not Be Empty   ${ns_name}
     ${rc}  ${stdout}=  Run and Return RC and Output  osm ns-op-list ${ns_name}
     log  ${stdout}
     log  ${rc}
@@ -259,10 +281,35 @@
 
     [Arguments]  ${ns_operation_id}
 
+    Should Not Be Empty   ${ns_operation_id}
     ${rc}  ${stdout}=  Run and Return RC and Output  osm ns-op-show ${ns_operation_id} --literal | yq .operationState | tr -d \\"
     log  ${stdout}
     Should Be Equal As Integers  ${rc}  ${success_return_code}
-    Should Contain  ${stdout}  COMPLETED  msg=Timeout waiting for ns-action with id ${ns_operation_id}  values=False
+    Should Contain  ${stdout}  COMPLETED  msg=The ns-action with id ${ns_operation_id} was not completed  values=False
+
+
+Check For NS Operation Failed
+    [Documentation]     Check wheter the status of the desired operation is "FAILED" or not.
+
+    [Arguments]  ${ns_operation_id}
+
+    Should Not Be Empty   ${ns_operation_id}
+    ${rc}  ${stdout}=  Run and Return RC and Output  osm ns-op-show ${ns_operation_id} --literal | yq .operationState | tr -d \\"
+    log  ${stdout}
+    Should Be Equal As Integers  ${rc}  ${success_return_code}
+    Should Contain  ${stdout}  FAILED  msg=Timeout waiting for ns-action with id ${ns_operation_id}  values=False
+
+
+Check For NS Operation Ended
+    [Documentation]     Check wheter the status of the desired operation is "FAILED" or "COMPLETED".
+
+    [Arguments]  ${ns_operation_id}
+
+    Should Not Be Empty   ${ns_operation_id}
+    ${rc}  ${stdout}=  Run and Return RC and Output  osm ns-op-show ${ns_operation_id} --literal | yq .operationState | tr -d \\"
+    log  ${stdout}
+    Should Be Equal As Integers  ${rc}  ${success_return_code}
+    Should Contain Any  ${stdout}  FAILED  COMPLETED  msg=Timeout waiting for ns-action with id ${ns_operation_id}  values=False
 
 
 Get Ns Vnfr Ids
@@ -270,6 +317,7 @@
 
     [Arguments]  ${ns_id}
 
+    Should Not Be Empty   ${ns_id}
     ${rc}  ${stdout}=  Run and Return RC and Output  osm vnf-list | grep ${ns_id} | awk '{print $2}' 2>&1
     Should Be Equal As Integers  ${rc}  ${success_return_code}  msg=${stdout}  values=False
     @{vdur} =  Split String  ${stdout}
@@ -281,6 +329,7 @@
 
     [Arguments]  ${vnf_id}
 
+    Should Not Be Empty   ${vnf_id}
     ${rc}  ${stdout}=  Run and Return RC and Output  osm vnf-show ${vnf_id} --literal | yq .vdur[].name | tr -d \\"
     Should Be Equal As Integers  ${rc}  ${success_return_code}  msg=${stdout}  values=False
     @{vdur} =  Split String  ${stdout}
@@ -292,6 +341,8 @@
 
     [Arguments]   ${vnf_id}   ${kdu_name}
 
+    Should Not Be Empty   ${vnf_id}
+    Should Not Be Empty   ${kdu_name}
     ${rc}  ${stdout}=  Run and Return RC and Output  osm vnf-show ${vnf_id} --kdu ${kdu_name} | yq .config.replicaCount | tr -d \\"
     Should Be Equal As Integers  ${rc}  ${success_return_code}  msg=${stdout}  values=False
     ${return} =   Set Variable If   '${stdout}' == 'null'   ${EMPTY}    ${stdout}
@@ -303,6 +354,7 @@
 
     [Arguments]   ${ns_name}
 
+    Should Not Be Empty   ${ns_name}
     ${rc}  ${stdout}=  Run and Return RC and Output  osm ns-show ${ns_name} --literal | yq ._admin.deployed.VCA[].application | tr -d \\"
     Should Be Equal As Integers  ${rc}  ${success_return_code}  msg=${stdout}  values=False
     @{app_names} =  Split String  ${stdout}
diff --git a/robot-systest/lib/openstack_lib.robot b/robot-systest/lib/openstack_lib.robot
index d544bb5..36e3f1d 100644
--- a/robot-systest/lib/openstack_lib.robot
+++ b/robot-systest/lib/openstack_lib.robot
@@ -103,3 +103,14 @@
     log   ${stdout}
     Should Be Equal As Integers   ${rc}   ${success_return_code}
     [Return]  ${stdout}
+
+
+Stop Server
+    [Documentation]     Stop a server
+    [Arguments]    ${server_id}
+
+    Should Not Be Empty   ${server_id}
+    ${rc}   ${stdout}=   Run and Return RC and Output   openstack server stop ${server_id}
+    log   ${stdout}
+    Should Be Equal As Integers   ${rc}   ${success_return_code}
+    [Return]  ${stdout}
diff --git a/robot-systest/lib/vnf_lib.robot b/robot-systest/lib/vnf_lib.robot
index 7b7d86d..d58c3dd 100644
--- a/robot-systest/lib/vnf_lib.robot
+++ b/robot-systest/lib/vnf_lib.robot
@@ -49,3 +49,26 @@
     Should Be Equal As Integers   ${rc}   ${success_return_code}   msg=${vm_vim_id}   values=False
     [Return]   ${vm_vim_id}
 
+
+Get Vnf Vdur IPs
+    [Documentation]     Return a list with the IP addresses of the VDU records of a VNF instance.
+
+    [Arguments]  ${vnf_id}
+
+    Should Not Be Empty   ${vnf_id}
+    ${rc}  ${stdout}=  Run and Return RC and Output  osm vnf-show ${vnf_id} --literal | yq '.vdur[].interfaces[]."ip-address"' | tr -d \\"
+    Should Be Equal As Integers  ${rc}  ${success_return_code}  msg=${stdout}  values=False
+    @{ip} =  Split String  ${stdout}
+    [Return]  @{ip}
+
+
+Get Vdu Attribute
+    [Documentation]     Return an attribute from VDU records selected by count_index of a VNF instance.
+
+    [Arguments]  ${vnf_id}   ${attribute}   ${count_index}=0
+
+    Should Not Be Empty   ${vnf_id}
+    Should Not Be Empty   ${attribute}
+    ${rc}  ${stdout}=  Run and Return RC and Output  osm vnf-show ${vnf_id} --literal | yq '.vdur[] | select(."count-index" == ${count_index})' | yq '."${attribute}"' | tr -d \\"
+    Should Be Equal As Integers  ${rc}  ${success_return_code}  msg=${stdout}  values=False
+    [Return]  ${stdout}