From: tierno Date: Mon, 30 Mar 2020 09:17:01 +0000 (+0000) Subject: fix wait option when operation fails X-Git-Tag: v7.1.0rc1^2~11 X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2Fosmclient.git;a=commitdiff_plain;h=bea7283d41dfaeb0eac8f940234dff9ae39e653b fix wait option when operation fails allow set a timeout for wait at ns,nsi,vim, wim, sdncontroller adding wait option to vnf-scale Change-Id: I7aa7aad8b678dcd19334a9d001e049be82476100 Signed-off-by: tierno --- diff --git a/osmclient/common/wait.py b/osmclient/common/wait.py index d8923ba..bb9a82a 100644 --- a/osmclient/common/wait.py +++ b/osmclient/common/wait.py @@ -20,8 +20,8 @@ OSM API handling for the '--wait' option from osmclient.common.exceptions import ClientException, NotFound import json -from time import sleep -import sys +from time import sleep, time +from sys import stderr # Declare a constant for each module, to allow customizing each timeout in the future TIMEOUT_GENERIC_OPERATION = 600 @@ -34,167 +34,150 @@ TIMEOUT_NS_OPERATION = 3600 POLLING_TIME_INTERVAL = 5 MAX_DELETE_ATTEMPTS = 3 + def _show_detailed_status(old_detailed_status, new_detailed_status): if new_detailed_status is not None and new_detailed_status != old_detailed_status: - sys.stderr.write("detailed-status: {}\n".format(new_detailed_status)) + stderr.write("detailed-status: {}\n".format(new_detailed_status)) return new_detailed_status else: return old_detailed_status + def _get_finished_states(entity): - # Note that the member name is either: - # 'operationState' (NS, NSI) - # '_admin.'operationalState' (VIM, WIM, SDN) - # For NS and NSI, 'operationState' may be one of: - # PROCESSING, COMPLETED,PARTIALLY_COMPLETED, FAILED_TEMP,FAILED,ROLLING_BACK,ROLLED_BACK - # For VIM, WIM, SDN: '_admin.operationalState' may be one of: - # operationalState: ENABLED, DISABLED, ERROR, PROCESSING + """ + Member name is either: + operationState' (NS, NSI) + '_admin.'operationalState' (VIM, WIM, SDN) + For NS and NSI, 'operationState' may be one of: + PROCESSING, COMPLETED,PARTIALLY_COMPLETED, FAILED_TEMP,FAILED,ROLLING_BACK,ROLLED_BACK + For VIM, WIM, SDN: '_admin.operationalState' may be one of: + ENABLED, DISABLED, ERROR, PROCESSING + + :param entity: can be NS, NSI, or other + :return: two tuples with status completed strings, status failed string + """ if entity == 'NS' or entity == 'NSI': - return ['COMPLETED', 'PARTIALLY_COMPLETED', 'FAILED_TEMP', 'FAILED'] + return ('COMPLETED', 'PARTIALLY_COMPLETED'), ('FAILED_TEMP', 'FAILED') else: - return ['ENABLED', 'ERROR'] + return ('ENABLED', ), ('ERROR', ) + def _get_operational_state(resp, entity): - # Note that the member name is either: - # 'operationState' (NS) - # 'operational-status' (NSI) - # '_admin.'operationalState' (other) + """ + The member name is either: + 'operationState' (NS) + 'operational-status' (NSI) + '_admin.'operationalState' (other) + :param resp: descriptor of the get response + :param entity: can be NS, NSI, or other + :return: status of the operation + """ if entity == 'NS' or entity == 'NSI': return resp.get('operationState') else: return resp.get('_admin', {}).get('operationalState') + def _op_has_finished(resp, entity): - # This function returns: - # 0 on success (operation has finished) - # 1 on pending (operation has not finished) - # -1 on error (bad response) - # - finished_states = _get_finished_states(entity) + """ + Indicates if operation has finished ok or is processing + :param resp: descriptor of the get response + :param entity: can be NS, NSI, or other + :return: + True on success (operation has finished) + False on pending (operation has not finished) + raise Exception if unexpected response, or ended with error + """ + finished_states_ok, finished_states_error = _get_finished_states(entity) if resp: - operationalState = _get_operational_state(resp, entity) - if operationalState: - if operationalState in finished_states: - return 0 - return 1 - return -1 - -def _get_detailed_status(resp, entity, detailed_status_deleted): - if detailed_status_deleted: - return detailed_status_deleted - if entity == 'NS' or entity == 'NSI': + op_state = _get_operational_state(resp, entity) + if op_state: + if op_state in finished_states_ok: + return True + elif op_state in finished_states_error: + raise ClientException("Operation failed with status '{}'".format(op_state)) + return False + raise ClientException('Unexpected response from server: {} '.format(resp)) + + +def _get_detailed_status(resp, entity): + """ + For VIM, WIM, SDN, 'detailed-status' is either: + - a leaf node to '_admin' (operations NOT supported) + - a leaf node of the Nth element in the list '_admin.operations[]' (operations supported by LCM and NBI) + :param resp: content of the get response + :param entity: can be NS, NSI, or other + :return: + """ + if entity in ('NS', 'NSI'): # For NS and NSI, 'detailed-status' is a JSON "root" member: return resp.get('detailed-status') else: - # For VIM, WIM, SDN, 'detailed-status' is either: - # - a leaf node to '_admin' (operations NOT supported) - # - a leaf node of the Nth element in the list '_admin.operations[]' (operations supported by LCM and NBI) - # https://osm.etsi.org/gerrit/#/c/7767 : LCM support for operations - # https://osm.etsi.org/gerrit/#/c/7734 : NBI support for current_operation ops = resp.get('_admin', {}).get('operations') - op_index = resp.get('_admin', {}).get('current_operation') - if ops and op_index: + current_op = resp.get('_admin', {}).get('current_operation') + if ops and current_op is not None: # Operations are supported, verify operation index - if isinstance(op_index, (int)) or op_index.isdigit(): - op_index = int(op_index) - if op_index > 0 and op_index < len(ops) and ops[op_index] and ops[op_index]["detailed-status"]: - return ops[op_index]["detailed-status"] + if isinstance(ops, dict) and current_op in ops: + return ops[current_op].get("detailed-status") + elif isinstance(ops, list) and isinstance(current_op, int) or current_op.isdigit(): + current_op = int(current_op) + if current_op >= 0 and current_op < len(ops) and ops[current_op] and ops[current_op]["detailed-status"]: + return ops[current_op]["detailed-status"] # operation index is either non-numeric or out-of-range return 'Unexpected error when getting detailed-status!' else: # Operations are NOT supported return resp.get('_admin', {}).get('detailed-status') -def _has_delete_error(resp, entity, deleteFlag, delete_attempts_left): - if deleteFlag and delete_attempts_left: - state = _get_operational_state(resp, entity) - if state and state == 'ERROR': - return True - return False def wait_for_status(entity_label, entity_id, timeout, apiUrlStatus, http_cmd, deleteFlag=False): - # Arguments: - # entity_label: String describing the entities using '--wait': - # 'NS', 'NSI', 'SDNC', 'VIM', 'WIM' - # entity_id: The ID for an existing entity, the operation ID for an entity to create. - # timeout: See section at top of this file for each value of TIMEOUT__OPERATION - # apiUrlStatus: The endpoint to get the Response including 'detailed-status' - # http_cmd: callback to HTTP command. - # Passing this callback as an argument avoids importing the 'http' module here. + """ + Wait until operation ends, making polling every 5s. Prints detailed status when it changes + :param entity_label: String describing the entities using '--wait': 'NS', 'NSI', 'SDNC', 'VIM', 'WIM' + :param entity_id: The ID for an existing entity, the operation ID for an entity to create. + :param timeout: Timeout in seconds + :param apiUrlStatus: The endpoint to get the Response including 'detailed-status' + :param http_cmd: callback to HTTP command. (Normally the get method) + :param deleteFlag: If this is a delete operation + :return: None, exception if operation fails or timeout + """ # Loop here until the operation finishes, or a timeout occurs. - time_left = timeout + time_to_finish = time() + timeout detailed_status = None - detailed_status_deleted = None - time_to_return = False - delete_attempts_left = MAX_DELETE_ATTEMPTS - wait_for_404 = False - try: - while True: + retries = 0 + max_retries = 1 + while True: + try: http_code, resp_unicode = http_cmd('{}/{}'.format(apiUrlStatus, entity_id)) - resp = '' - if resp_unicode: - resp = json.loads(resp_unicode) - # print('HTTP CODE: {}'.format(http_code)) - # print('RESP: {}'.format(resp)) - # print('URL: {}/{}'.format(apiUrlStatus, entity_id)) - if deleteFlag and http_code == 404: - # In case of deletion, '404 Not Found' means successfully deleted - # Display 'detailed-status: Deleted' and return - time_to_return = True - detailed_status_deleted = 'Deleted' - elif deleteFlag and http_code in (200, 201, 202, 204): - # In case of deletion and HTTP Status = 20* OK, deletion may be PROCESSING or COMPLETED - # If this is the case, we should keep on polling until 404 (deleted) is returned. - wait_for_404 = True - elif http_code not in (200, 201, 202, 204): - raise ClientException(str(resp)) - if not time_to_return: - # Get operation status - op_status = _op_has_finished(resp, entity_label) - if op_status == -1: - # An error occurred - raise ClientException('unexpected response from server - {} '.format( - str(resp))) - elif op_status == 0: - # If there was an error upon deletion, try again to delete the same instance - # If the error is the same, there is probably nothing we can do but exit with error. - # If the error is different (i.e. 404), the instance was probably already corrupt, that is, - # operation(al)State was probably ERROR before deletion. - # In such a case, even if the previous state was ERROR, the deletion was successful, - # so detailed-status should be set to Deleted. - if _has_delete_error(resp, entity_label, deleteFlag, delete_attempts_left): - delete_attempts_left -= 1 - else: - # Operation has finished, either with success or error - if deleteFlag: - delete_attempts_left -= 1 - if not wait_for_404 and delete_attempts_left < MAX_DELETE_ATTEMPTS: - time_to_return = True - else: - time_to_return = True - new_detailed_status = _get_detailed_status(resp, entity_label, detailed_status_deleted) - # print('DETAILED-STATUS: {}'.format(new_detailed_status)) - # print('DELETE-ATTEMPTS-LEFT: {}'.format(delete_attempts_left)) - if not new_detailed_status: - new_detailed_status = 'In progress' - # TODO: Change LCM to provide detailed-status more up to date - # At the moment of this writing, 'detailed-status' may return different strings - # from different resources: - # /nslcm/v1/ns_lcm_op_occs/ ---> '' - # /nslcm/v1/ns_instances_content/ ---> 'deleting charms' - detailed_status = _show_detailed_status(detailed_status, new_detailed_status) - if time_to_return: + retries = 0 + except NotFound: + if deleteFlag: + _show_detailed_status(detailed_status, 'Deleted') return - time_left -= POLLING_TIME_INTERVAL + raise + except ClientException: + if retries >= max_retries or time() < time_to_finish: + raise + retries += 1 sleep(POLLING_TIME_INTERVAL) - if time_left <= 0: - # There was a timeout, so raise an exception - raise ClientException('operation timeout, waited for {} seconds'.format(timeout)) - except ClientException as exc: - if deleteFlag and isinstance(exc, NotFound): + continue + + resp = '' + if resp_unicode: + resp = json.loads(resp_unicode) + + new_detailed_status = _get_detailed_status(resp, entity_label) + # print('DETAILED-STATUS: {}'.format(new_detailed_status)) + if not new_detailed_status: + new_detailed_status = 'In progress' + detailed_status = _show_detailed_status(detailed_status, new_detailed_status) + + # Get operation status + if _op_has_finished(resp, entity_label): return - message = "Operation failed for {}:\nerror:\n{}".format( - entity_label, - str(exc)) - raise ClientException(message) + + if time() >= time_to_finish: + # There was a timeout, so raise an exception + raise ClientException('operation timeout after {} seconds'.format(timeout)) + sleep(POLLING_TIME_INTERVAL) diff --git a/osmclient/scripts/osm.py b/osmclient/scripts/osm.py index e1dbb8e..6e0dc9e 100755 --- a/osmclient/scripts/osm.py +++ b/osmclient/scripts/osm.py @@ -3633,13 +3633,16 @@ def ns_action(ctx, @click.option('--scaling-group', prompt=True, help="scaling-group-descriptor name to use") @click.option('--scale-in', default=False, is_flag=True, help="performs a scale in operation") @click.option('--scale-out', default=False, is_flag=True, help="performs a scale out operation (by default)") +@click.option('--wait', required=False, default=False, is_flag=True, + help='do not return the control immediately, but keep it until the operation is completed, or timeout') @click.pass_context def vnf_scale(ctx, ns_name, vnf_name, scaling_group, scale_in, - scale_out): + scale_out, + wait): """ Executes a VNF scale (adding/removing VDUs) @@ -3652,7 +3655,7 @@ def vnf_scale(ctx, check_client_version(ctx.obj, ctx.command.name) if not scale_in and not scale_out: scale_out = True - ctx.obj.ns.scale_vnf(ns_name, vnf_name, scaling_group, scale_in, scale_out) + ctx.obj.ns.scale_vnf(ns_name, vnf_name, scaling_group, scale_in, scale_out, wait) # except ClientException as e: # print(str(e)) # exit(1) diff --git a/osmclient/sol005/ns.py b/osmclient/sol005/ns.py index 2fce5ab..f4b5b89 100644 --- a/osmclient/sol005/ns.py +++ b/osmclient/sol005/ns.py @@ -40,15 +40,17 @@ class Ns(object): self._apiVersion, self._apiResource) # NS '--wait' option - def _wait(self, id, deleteFlag=False): + def _wait(self, id, wait_time, deleteFlag=False): self._logger.debug("") # Endpoint to get operation status apiUrlStatus = '{}{}{}'.format(self._apiName, self._apiVersion, '/ns_lcm_op_occs') # Wait for status for NS instance creation/update/deletion + if isinstance(wait_time, bool): + wait_time = WaitForStatus.TIMEOUT_NS_OPERATION WaitForStatus.wait_for_status( 'NS', str(id), - WaitForStatus.TIMEOUT_NS_OPERATION, + wait_time, apiUrlStatus, self._http.get2_cmd, deleteFlag=deleteFlag) @@ -101,6 +103,18 @@ class Ns(object): raise NotFound("ns '{}' not found".format(name)) def delete(self, name, force=False, config=None, wait=False): + """ + Deletes a Network Service (NS) + :param name: name of network service + :param force: set force. Direct deletion without cleaning at VIM + :param config: parameters of deletion, as: + autoremove: Bool (default True) + timeout_ns_terminate: int + skip_terminate_primitives: Bool (default False) to not exec the terminate primitives + :param wait: Make synchronous. Wait until deletion is completed: + False to not wait (by default), True to wait a standard time, or int (time to wait) + :return: None. Exception if fail + """ self._logger.debug("") ns = self.get(name) querystring_list = [] @@ -123,7 +137,7 @@ class Ns(object): if wait and resp: resp = json.loads(resp) # For the 'delete' operation, '_id' is used - self._wait(resp.get('_id'), deleteFlag=True) + self._wait(resp.get('_id'), wait, deleteFlag=True) else: print('Deletion in progress') elif http_code == 204: @@ -250,7 +264,7 @@ class Ns(object): resp)) if wait: # Wait for status for NS instance creation - self._wait(resp.get('nslcmop_id')) + self._wait(resp.get('nslcmop_id'), wait) print(resp['id']) return resp['id'] #else: @@ -364,7 +378,7 @@ class Ns(object): if wait: # Wait for status for NS instance action # For the 'action' operation, 'id' is used - self._wait(resp.get('id')) + self._wait(resp.get('id'), wait) return resp['id'] #else: # msg = "" @@ -389,10 +403,12 @@ class Ns(object): op_data={} op_data["scaleType"] = "SCALE_VNF" op_data["scaleVnfData"] = {} - if scale_in: + if scale_in and not scale_out: op_data["scaleVnfData"]["scaleVnfType"] = "SCALE_IN" - else: + elif not scale_in and scale_out: op_data["scaleVnfData"]["scaleVnfType"] = "SCALE_OUT" + else: + raise ClientException("you must set either 'scale_in' or 'scale_out'") op_data["scaleVnfData"]["scaleByStepData"] = { "member-vnf-index": vnf_name, "scaling-group-descriptor": scaling_group, diff --git a/osmclient/sol005/nsi.py b/osmclient/sol005/nsi.py index 4b522a8..4671441 100644 --- a/osmclient/sol005/nsi.py +++ b/osmclient/sol005/nsi.py @@ -40,16 +40,18 @@ class Nsi(object): self._apiVersion, self._apiResource) # NSI '--wait' option - def _wait(self, id, deleteFlag=False): + def _wait(self, id, wait_time, deleteFlag=False): self._logger.debug("") self._client.get_token() # Endpoint to get operation status apiUrlStatus = '{}{}{}'.format(self._apiName, self._apiVersion, '/nsi_lcm_op_occs') # Wait for status for NSI instance creation/update/deletion + if isinstance(wait_time, bool): + wait_time = WaitForStatus.TIMEOUT_NSI_OPERATION WaitForStatus.wait_for_status( 'NSI', str(id), - WaitForStatus.TIMEOUT_NSI_OPERATION, + wait_time, apiUrlStatus, self._http.get2_cmd, deleteFlag=deleteFlag) @@ -116,7 +118,7 @@ class Nsi(object): resp = json.loads(resp) # Wait for status for NSI instance deletion # For the 'delete' operation, '_id' is used - self._wait(resp.get('_id'), deleteFlag=True) + self._wait(resp.get('_id'), wait, deleteFlag=True) else: print('Deletion in progress') elif http_code == 204: @@ -240,7 +242,7 @@ class Nsi(object): resp)) if wait: # Wait for status for NSI instance creation - self._wait(resp.get('nsilcmop_id')) + self._wait(resp.get('nsilcmop_id'), wait) print(resp['id']) #else: # msg = "" diff --git a/osmclient/sol005/sdncontroller.py b/osmclient/sol005/sdncontroller.py index b02632e..5b85ef4 100644 --- a/osmclient/sol005/sdncontroller.py +++ b/osmclient/sol005/sdncontroller.py @@ -39,16 +39,18 @@ class SdnController(object): self._apiVersion, self._apiResource) # SDNC '--wait' option - def _wait(self, id, deleteFlag=False): + def _wait(self, id, wait_time, deleteFlag=False): self._logger.debug("") self._client.get_token() # Endpoint to get operation status apiUrlStatus = '{}{}{}'.format(self._apiName, self._apiVersion, '/sdns') # Wait for status for SDN instance creation/update/deletion + if isinstance(wait_time, bool): + wait_time = WaitForStatus.TIMEOUT_SDNC_OPERATION WaitForStatus.wait_for_status( 'SDNC', str(id), - WaitForStatus.TIMEOUT_SDNC_OPERATION, + wait_time, apiUrlStatus, self._http.get2_cmd, deleteFlag=deleteFlag) @@ -82,7 +84,7 @@ class SdnController(object): resp)) if wait: # Wait for status for SDNC instance creation - self._wait(resp.get('id')) + self._wait(resp.get('id'), wait) print(resp['id']) #else: # msg = "" @@ -110,7 +112,7 @@ class SdnController(object): # Use the previously obtained id instead. wait_id = sdnc_id_for_wait # Wait for status for VI instance update - self._wait(wait_id) + self._wait(wait_id, wait) # else: # pass #else: @@ -137,7 +139,7 @@ class SdnController(object): if http_code == 202: if wait: # Wait for status for SDNC instance deletion - self._wait(sdnc_id_for_wait, deleteFlag=True) + self._wait(sdnc_id_for_wait, wait, deleteFlag=True) else: print('Deletion in progress') elif http_code == 204: diff --git a/osmclient/sol005/vim.py b/osmclient/sol005/vim.py index 16c3615..3441161 100644 --- a/osmclient/sol005/vim.py +++ b/osmclient/sol005/vim.py @@ -39,16 +39,18 @@ class Vim(object): self._apiVersion, self._apiResource) # VIM '--wait' option - def _wait(self, id, deleteFlag=False): + def _wait(self, id, wait_time, deleteFlag=False): self._logger.debug("") self._client.get_token() # Endpoint to get operation status apiUrlStatus = '{}{}{}'.format(self._apiName, self._apiVersion, '/vim_accounts') # Wait for status for VIM instance creation/deletion + if isinstance(wait_time, bool): + wait_time = WaitForStatus.TIMEOUT_VIM_OPERATION WaitForStatus.wait_for_status( 'VIM', str(id), - WaitForStatus.TIMEOUT_VIM_OPERATION, + wait_time, apiUrlStatus, self._http.get2_cmd, deleteFlag=deleteFlag) @@ -102,7 +104,7 @@ class Vim(object): resp)) if wait: # Wait for status for VIM instance creation - self._wait(resp.get('id')) + self._wait(resp.get('id'), wait) print(resp['id']) #else: # msg = "" @@ -148,7 +150,7 @@ class Vim(object): # Use the previously obtained id instead. wait_id = vim_id_for_wait # Wait for status for VI instance update - self._wait(wait_id) + self._wait(wait_id, wait) # else: # pass #else: @@ -201,7 +203,7 @@ class Vim(object): resp = json.loads(resp) wait_id = resp.get('id') # Wait for status for VIM account deletion - self._wait(wait_id, deleteFlag=True) + self._wait(wait_id, wait, deleteFlag=True) else: print('Deletion in progress') elif http_code == 204: diff --git a/osmclient/sol005/wim.py b/osmclient/sol005/wim.py index f9d2431..0ba8ab4 100644 --- a/osmclient/sol005/wim.py +++ b/osmclient/sol005/wim.py @@ -39,16 +39,18 @@ class Wim(object): self._apiVersion, self._apiResource) # WIM '--wait' option - def _wait(self, id, deleteFlag=False): + def _wait(self, id, wait_time, deleteFlag=False): self._logger.debug("") self._client.get_token() # Endpoint to get operation status apiUrlStatus = '{}{}{}'.format(self._apiName, self._apiVersion, '/wim_accounts') # Wait for status for WIM instance creation/deletion + if isinstance(wait_time, bool): + wait_time = WaitForStatus.TIMEOUT_WIM_OPERATION WaitForStatus.wait_for_status( 'WIM', str(id), - WaitForStatus.TIMEOUT_WIM_OPERATION, + wait_time, apiUrlStatus, self._http.get2_cmd, deleteFlag=deleteFlag) @@ -96,7 +98,7 @@ class Wim(object): resp)) if wait: # Wait for status for WIM instance creation - self._wait(resp.get('id')) + self._wait(resp.get('id'), wait) print(resp['id']) #else: # msg = "" @@ -135,7 +137,7 @@ class Wim(object): # Use the previously obtained id instead. wait_id = wim_id_for_wait # Wait for status for WIM instance update - self._wait(wait_id) + self._wait(wait_id, wait) # else: # pass #else: @@ -190,7 +192,7 @@ class Wim(object): resp = json.loads(resp) wait_id = resp.get('id') # Wait for status for WIM account deletion - self._wait(wait_id, deleteFlag=True) + self._wait(wait_id, wait, deleteFlag=True) else: print('Deletion in progress') elif http_code == 204: