await self.disconnect_model(model)
await self.disconnect_controller(controller)
+ async def resolve(self, model_name: str):
+
+ controller = await self.get_controller()
+ model = await self.get_model(controller, model_name)
+ all_units_active = False
+ try:
+ while not all_units_active:
+ all_units_active = True
+ for application_name, application in model.applications.items():
+ if application.status == "error":
+ for unit in application.units:
+ if unit.workload_status == "error":
+ self.log.debug(
+ "Model {}, Application {}, Unit {} in error state, resolving".format(
+ model_name, application_name, unit.entity_id
+ )
+ )
+ try:
+ await unit.resolved(retry=False)
+ all_units_active = False
+ except Exception:
+ pass
+
+ if not all_units_active:
+ await asyncio.sleep(5)
+ finally:
+ await self.disconnect_model(model)
+ await self.disconnect_controller(controller)
+
async def scale_application(
self,
model_name: str,
try:
await model.add_relation(endpoint_1, endpoint_2)
except juju.errors.JujuAPIError as e:
- if "not found" in e.message:
+ if self._relation_is_not_found(e):
self.log.warning("Relation not found: {}".format(e.message))
return
- if "already exists" in e.message:
+ if self._relation_already_exist(e):
self.log.warning("Relation already exists: {}".format(e.message))
return
# another exception, raise it
await self.disconnect_model(model)
await self.disconnect_controller(controller)
+ def _relation_is_not_found(self, juju_error):
+ text = "not found"
+ return (text in juju_error.message) or (
+ juju_error.error_code and text in juju_error.error_code
+ )
+
+ def _relation_already_exist(self, juju_error):
+ text = "already exists"
+ return (text in juju_error.message) or (
+ juju_error.error_code and text in juju_error.error_code
+ )
+
async def offer(self, endpoint: RelationEndpoint) -> Offer:
"""
Create an offer from a RelationEndpoint
await self._destroy_pending_machines(model, only_manual=True)
await self.disconnect_model(model)
- await self._destroy_model(
- model_name,
- controller,
+ await asyncio.wait_for(
+ self._destroy_model(model_name, controller),
timeout=total_timeout,
)
except Exception as e:
await self.disconnect_controller(controller)
async def _destroy_model(
- self, model_name: str, controller: Controller, timeout: float = 1800
+ self,
+ model_name: str,
+ controller: Controller,
):
"""
Destroy model from controller
"""
self.log.debug(f"Destroying model {model_name}")
- async def _destroy_model_loop(model_name: str, controller: Controller):
- while await self.model_exists(model_name, controller=controller):
+ async def _destroy_model_gracefully(model_name: str, controller: Controller):
+ self.log.info(f"Gracefully deleting model {model_name}")
+ resolved = False
+ while model_name in await controller.list_models():
+ if not resolved:
+ await self.resolve(model_name)
+ resolved = True
+ await controller.destroy_model(model_name, destroy_storage=True)
+
+ await asyncio.sleep(5)
+ self.log.info(f"Model {model_name} deleted gracefully")
+
+ async def _destroy_model_forcefully(model_name: str, controller: Controller):
+ self.log.info(f"Forcefully deleting model {model_name}")
+ while model_name in await controller.list_models():
await controller.destroy_model(
- model_name, destroy_storage=True, force=True, max_wait=0
+ model_name, destroy_storage=True, force=True, max_wait=60
)
await asyncio.sleep(5)
+ self.log.info(f"Model {model_name} deleted forcefully")
try:
- await asyncio.wait_for(
- _destroy_model_loop(model_name, controller), timeout=timeout
- )
- except asyncio.TimeoutError:
- raise Exception(
- "Timeout waiting for model {} to be destroyed".format(model_name)
- )
+ try:
+ await asyncio.wait_for(
+ _destroy_model_gracefully(model_name, controller), timeout=120
+ )
+ except asyncio.TimeoutError:
+ await _destroy_model_forcefully(model_name, controller)
except juju.errors.JujuError as e:
if any("has been removed" in error for error in e.errors):
return
+ if any("model not found" in error for error in e.errors):
+ return
raise e
async def destroy_application(