From: aktas Date: Thu, 29 Jul 2021 14:41:40 +0000 (+0300) Subject: Bug 1609 fix X-Git-Tag: v10.0.3~3 X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=refs%2Fchanges%2F12%2F11112%2F2;p=osm%2FN2VC.git Bug 1609 fix This patch changes the behaviour of native charm deployments. it won't deploy another application for the same vnf or vdu charm at initial deployment or scaling process. It scales the application. Change-Id: I3fc52a5ddb0cb7cb16937bc12cf343f7d869c9ee Signed-off-by: aktas (cherry picked from commit fa02f8a90b7fe1e1b7a80feedef4132bef1ca3e4) --- diff --git a/.gitignore b/.gitignore index 191b24e..e7b66fb 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,4 @@ lib64 pyvenv.cfg share/ venv/ +.idea diff --git a/n2vc/exceptions.py b/n2vc/exceptions.py index 1f48661..9867cb9 100644 --- a/n2vc/exceptions.py +++ b/n2vc/exceptions.py @@ -111,6 +111,18 @@ class N2VCNotFound(N2VCException): return "<{}> Not found: {}".format(type(self), super().__str__()) +class N2VCApplicationExists(N2VCException): + """ + Application Exists + """ + + def __init__(self, message: str = ""): + N2VCException.__init__(self, message=message) + + def __str__(self): + return "<{}> Application Exists: {}".format(type(self), super().__str__()) + + class JujuError(N2VCException): """ Juju Error diff --git a/n2vc/libjuju.py b/n2vc/libjuju.py index 9dd8f3b..9961e19 100644 --- a/n2vc/libjuju.py +++ b/n2vc/libjuju.py @@ -566,6 +566,147 @@ class Libjuju: await self.disconnect_model(model) await self.disconnect_controller(controller) + async def add_unit( + self, + application_name: str, + model_name: str, + machine_id: str, + db_dict: dict = None, + progress_timeout: float = None, + total_timeout: float = None, + ): + """Add unit + + :param: application_name: Application name + :param: model_name: Model name + :param: machine_id Machine id + :param: db_dict: Dictionary with data of the DB to write the updates + :param: progress_timeout: Maximum time between two updates in the model + :param: total_timeout: Timeout for the entity to be active + + :return: None + """ + + model = None + controller = await self.get_controller() + try: + model = await self.get_model(controller, model_name) + application = self._get_application(model, application_name) + + if application is not None: + + # Checks if the given machine id in the model, + # otherwise function raises an error + _machine, _series = self._get_machine_info(model, machine_id) + + self.log.debug( + "Adding unit (machine {}) to application {} in model ~{}".format( + machine_id, application_name, model_name + ) + ) + + await application.add_unit(to=machine_id) + + await JujuModelWatcher.wait_for( + model=model, + entity=application, + progress_timeout=progress_timeout, + total_timeout=total_timeout, + db_dict=db_dict, + n2vc=self.n2vc, + vca_id=self.vca_connection._vca_id, + ) + self.log.debug( + "Unit is added to application {} in model {}".format( + application_name, model_name + ) + ) + else: + raise JujuApplicationNotFound( + "Application {} not exists".format(application_name) + ) + finally: + if model: + await self.disconnect_model(model) + await self.disconnect_controller(controller) + + async def destroy_unit( + self, + application_name: str, + model_name: str, + machine_id: str, + total_timeout: float = None, + ): + """Destroy unit + + :param: application_name: Application name + :param: model_name: Model name + :param: machine_id Machine id + :param: db_dict: Dictionary with data of the DB to write the updates + :param: total_timeout: Timeout for the entity to be active + + :return: None + """ + + model = None + controller = await self.get_controller() + try: + model = await self.get_model(controller, model_name) + application = self._get_application(model, application_name) + + if application is None: + raise JujuApplicationNotFound( + "Application not found: {} (model={})".format( + application_name, model_name + ) + ) + + unit = self._get_unit(application, machine_id) + if not unit: + raise JujuError( + "A unit with machine id {} not in available units".format( + machine_id + ) + ) + + unit_name = unit.name + + self.log.debug( + "Destroying unit {} from application {} in model {}".format( + unit_name, application_name, model_name + ) + ) + await application.destroy_unit(unit_name) + + self.log.debug( + "Waiting for unit {} to be destroyed in application {} (model={})...".format( + unit_name, application_name, model_name + ) + ) + + # TODO: Add functionality in the Juju watcher to replace this kind of blocks + if total_timeout is None: + total_timeout = 3600 + end = time.time() + total_timeout + while time.time() < end: + if not self._get_unit(application, machine_id): + self.log.debug( + "The unit {} was destroyed in application {} (model={}) ".format( + unit_name, application_name, model_name + ) + ) + return + await asyncio.sleep(5) + self.log.debug( + "Unit {} is destroyed from application {} in model {}".format( + unit_name, application_name, model_name + ) + ) + finally: + if model: + await self.disconnect_model(model) + await self.disconnect_controller(controller) + async def deploy_charm( self, application_name: str, @@ -608,16 +749,10 @@ class Libjuju: model = await self.get_model(controller, model_name) try: - application = None if application_name not in model.applications: if machine_id is not None: - if machine_id not in model.machines: - msg = "Machine {} not found in model".format(machine_id) - self.log.error(msg=msg) - raise JujuMachineNotFound(msg) - machine = model.machines[machine_id] - series = machine.series + machine, series = self._get_machine_info(model, machine_id) application = await model.deploy( entity_url=path, @@ -751,12 +886,47 @@ class Libjuju: if model.applications and application_name in model.applications: return model.applications[application_name] + def _get_unit(self, application: Application, machine_id: str) -> Unit: + """Get unit + + :param: application: Application object + :param: machine_id: Machine id + + :return: Unit + """ + unit = None + for u in application.units: + if u.machine_id == machine_id: + unit = u + break + return unit + + def _get_machine_info( + self, + model, + machine_id: str, + ) -> (str, str): + """Get machine info + + :param: model: Model object + :param: machine_id: Machine id + + :return: (str, str): (machine, series) + """ + if machine_id not in model.machines: + msg = "Machine {} not found in model".format(machine_id) + self.log.error(msg=msg) + raise JujuMachineNotFound(msg) + machine = model.machines[machine_id] + return machine, machine.series + async def execute_action( self, application_name: str, model_name: str, action_name: str, db_dict: dict = None, + machine_id: str = None, progress_timeout: float = None, total_timeout: float = None, **kwargs, @@ -767,6 +937,7 @@ class Libjuju: :param: model_name: Model name :param: action_name: Name of the action :param: db_dict: Dictionary with data of the DB to write the updates + :param: machine_id Machine id :param: progress_timeout: Maximum time between two updates in the model :param: total_timeout: Timeout for the entity to be active @@ -789,14 +960,31 @@ class Libjuju: ) if application is None: raise JujuApplicationNotFound("Cannot execute action") - - # Get leader unit # Racing condition: # Ocassionally, self._get_leader_unit() will return None # because the leader elected hook has not been triggered yet. # Therefore, we are doing some retries. If it happens again, # re-open bug 1236 - unit = await self._get_leader_unit(application) + if machine_id is None: + unit = await self._get_leader_unit(application) + self.log.debug( + "Action {} is being executed on the leader unit {}".format( + action_name, unit.name + ) + ) + else: + unit = self._get_unit(application, machine_id) + if not unit: + raise JujuError( + "A unit with machine id {} not in available units".format( + machine_id + ) + ) + self.log.debug( + "Action {} is being executed on {} unit".format( + action_name, unit.name + ) + ) actions = await application.get_actions() @@ -1332,3 +1520,27 @@ class Libjuju: return (await facade.Credential(params)).results finally: await self.disconnect_controller(controller) + + async def check_application_exists(self, model_name, application_name) -> bool: + """Check application exists + + :param: model_name: Model Name + :param: application_name: Application Name + + :return: Boolean + """ + + model = None + controller = await self.get_controller() + try: + model = await self.get_model(controller, model_name) + self.log.debug( + "Checking if application {} exists in model {}".format( + application_name, model_name + ) + ) + return self._get_application(model, application_name) is not None + finally: + if model: + await self.disconnect_model(model) + await self.disconnect_controller(controller) diff --git a/n2vc/n2vc_juju_conn.py b/n2vc/n2vc_juju_conn.py index eed2f30..2c94328 100644 --- a/n2vc/n2vc_juju_conn.py +++ b/n2vc/n2vc_juju_conn.py @@ -29,6 +29,8 @@ from n2vc.exceptions import ( N2VCException, N2VCConnectionException, N2VCExecutionException, + N2VCApplicationExists, + JujuApplicationExists, # N2VCNotFound, MethodNotImplemented, ) @@ -37,6 +39,7 @@ from n2vc.n2vc_conn import obj_to_dict, obj_to_yaml from n2vc.libjuju import Libjuju from n2vc.store import MotorStore from n2vc.vca.connection import get_connection +from retrying_async import retry class N2VCJujuConnector(N2VCConnector): @@ -364,6 +367,9 @@ class N2VCJujuConnector(N2VCConnector): return ee_id + # In case of native_charm is being deployed, if JujuApplicationExists error happens + # it will try to add_unit + @retry(attempts=3, delay=5, retry_exceptions=(N2VCApplicationExists,)) async def install_configuration_sw( self, ee_id: str, @@ -374,6 +380,8 @@ class N2VCJujuConnector(N2VCConnector): config: dict = None, num_units: int = 1, vca_id: str = None, + scaling_out: bool = False, + vca_type: str = None, ): """ Install the software inside the execution environment identified by ee_id @@ -395,6 +403,8 @@ class N2VCJujuConnector(N2VCConnector): :param: config: Dictionary with deployment config information. :param: num_units: Number of units to deploy of a particular charm. :param: vca_id: VCA ID + :param: scaling_out: Boolean to indicate if it is a scaling out operation + :param: vca_type: VCA type """ self.log.info( @@ -453,20 +463,36 @@ class N2VCJujuConnector(N2VCConnector): full_path = self.fs.path + "/" + artifact_path try: - await libjuju.deploy_charm( - model_name=model_name, - application_name=application_name, - path=full_path, - machine_id=machine_id, - db_dict=db_dict, - progress_timeout=progress_timeout, - total_timeout=total_timeout, - config=config, - num_units=num_units, + if vca_type == "native_charm" and await libjuju.check_application_exists( + model_name, application_name + ): + await libjuju.add_unit( + application_name=application_name, + model_name=model_name, + machine_id=machine_id, + db_dict=db_dict, + progress_timeout=progress_timeout, + total_timeout=total_timeout, + ) + else: + await libjuju.deploy_charm( + model_name=model_name, + application_name=application_name, + path=full_path, + machine_id=machine_id, + db_dict=db_dict, + progress_timeout=progress_timeout, + total_timeout=total_timeout, + config=config, + num_units=num_units, + ) + except JujuApplicationExists as e: + raise N2VCApplicationExists( + message="Error deploying charm into ee={} : {}".format(ee_id, e.message) ) except Exception as e: raise N2VCException( - message="Error desploying charm into ee={} : {}".format(ee_id, e) + message="Error deploying charm into ee={} : {}".format(ee_id, e) ) self.log.info("Configuration sw installed") @@ -819,6 +845,7 @@ class N2VCJujuConnector(N2VCConnector): db_dict: dict = None, total_timeout: float = None, scaling_in: bool = False, + vca_type: str = None, vca_id: str = None, ): """ @@ -830,7 +857,8 @@ class N2VCJujuConnector(N2VCConnector): e.g. {collection: "nsrs", filter: {_id: , path: "_admin.deployed.VCA.3"} :param: total_timeout: Total timeout - :param: scaling_in: Boolean to indicate if is it a scaling in operation + :param: scaling_in: Boolean to indicate if it is a scaling in operation + :param: vca_type: VCA type :param: vca_id: VCA ID """ self.log.info("Deleting execution environment ee_id={}".format(ee_id)) @@ -842,17 +870,25 @@ class N2VCJujuConnector(N2VCConnector): message="ee_id is mandatory", bad_args=["ee_id"] ) - model_name, application_name, _machine_id = self._get_ee_id_components( + model_name, application_name, machine_id = self._get_ee_id_components( ee_id=ee_id ) try: if not scaling_in: # destroy the model - # TODO: should this be removed? await libjuju.destroy_model( model_name=model_name, total_timeout=total_timeout, ) + elif vca_type == "native_charm" and scaling_in: + # destroy the unit in the application + await libjuju.destroy_unit( + application_name=application_name, + model_name=model_name, + machine_id=machine_id, + db_dict=db_dict, + total_timeout=total_timeout, + ) else: # destroy the application await libjuju.destroy_application( @@ -878,6 +914,7 @@ class N2VCJujuConnector(N2VCConnector): progress_timeout: float = None, total_timeout: float = None, vca_id: str = None, + vca_type: str = None, ) -> str: """ Execute a primitive in the execution environment @@ -896,6 +933,7 @@ class N2VCJujuConnector(N2VCConnector): :param: progress_timeout: Progress timeout :param: total_timeout: Total timeout :param: vca_id: VCA ID + :param: vca_type: VCA type :returns str: primitive result, if ok. It raises exceptions in case of fail """ @@ -922,8 +960,12 @@ class N2VCJujuConnector(N2VCConnector): ( model_name, application_name, - _machine_id, + machine_id, ) = N2VCJujuConnector._get_ee_id_components(ee_id=ee_id) + # To run action on the leader unit in libjuju.execute_action function, + # machine_id must be set to None if vca_type is not native_charm + if vca_type != "native_charm": + machine_id = None except Exception: raise N2VCBadArgumentsException( message="ee_id={} is not a valid execution environment id".format( @@ -1003,6 +1045,7 @@ class N2VCJujuConnector(N2VCConnector): application_name=application_name, action_name=primitive_name, db_dict=db_dict, + machine_id=machine_id, progress_timeout=progress_timeout, total_timeout=total_timeout, **params_dict diff --git a/n2vc/tests/unit/test_libjuju.py b/n2vc/tests/unit/test_libjuju.py index 3447340..02f2b21 100644 --- a/n2vc/tests/unit/test_libjuju.py +++ b/n2vc/tests/unit/test_libjuju.py @@ -20,7 +20,12 @@ import juju import kubernetes from juju.errors import JujuAPIError import logging -from .utils import FakeMachine, FakeApplication +from .utils import ( + FakeApplication, + FakeMachine, + FakeManualMachine, + FakeUnit, +) from n2vc.libjuju import Libjuju from n2vc.exceptions import ( JujuControllerFailedConnecting, @@ -824,7 +829,7 @@ class ExecuteActionTest(LibjujuTestCase): mock_disconnect_controller.assert_called() mock_disconnect_model.assert_called() - def test_succesful_exec( + def test_successful_exec( self, mock_get_action_status, mock_get_action_output, @@ -1966,7 +1971,7 @@ class GetUnitNumberTest(LibjujuTestCase): def setUp(self): super(GetUnitNumberTest, self).setUp() - def test_succesful_get_unit_number( + def test_successful_get_unit_number( self, mock_get_applications, ): @@ -1983,3 +1988,287 @@ class GetUnitNumberTest(LibjujuTestCase): model = juju.model.Model() result = self.libjuju._get_application_count(model, "app") self.assertEqual(result, None) + + +@asynctest.mock.patch("juju.model.Model.machines", new_callable=asynctest.PropertyMock) +class GetMachineInfoTest(LibjujuTestCase): + def setUp(self): + super(GetMachineInfoTest, self).setUp() + + def test_successful( + self, + mock_machines, + ): + machine_id = "existing_machine" + model = juju.model.Model() + mock_machines.return_value = {"existing_machine": FakeManualMachine()} + machine, series = self.libjuju._get_machine_info( + machine_id=machine_id, + model=model, + ) + self.assertIsNotNone(machine, series) + + def test_exception( + self, + mock_machines, + ): + machine_id = "not_existing_machine" + machine = series = None + model = juju.model.Model() + mock_machines.return_value = {"existing_machine": FakeManualMachine()} + with self.assertRaises(JujuMachineNotFound): + machine, series = self.libjuju._get_machine_info( + machine_id=machine_id, + model=model, + ) + self.assertIsNone(machine, series) + + +class GetUnitTest(LibjujuTestCase): + def setUp(self): + super(GetUnitTest, self).setUp() + + def test_successful(self): + result = self.libjuju._get_unit(FakeApplication(), "existing_machine_id") + self.assertIsInstance(result, FakeUnit) + + def test_return_none(self): + result = self.libjuju._get_unit(FakeApplication(), "not_existing_machine_id") + self.assertIsNone(result) + + +@asynctest.mock.patch("n2vc.libjuju.Libjuju.get_controller") +@asynctest.mock.patch("n2vc.libjuju.Libjuju.get_model") +@asynctest.mock.patch("n2vc.libjuju.Libjuju.disconnect_model") +@asynctest.mock.patch("n2vc.libjuju.Libjuju.disconnect_controller") +@asynctest.mock.patch("n2vc.libjuju.Libjuju._get_application") +class CheckApplicationExists(LibjujuTestCase): + def setUp(self): + super(CheckApplicationExists, self).setUp() + + def test_successful( + self, + mock_get_application, + mock_disconnect_controller, + mock_disconnect_model, + mock_get_model, + mock_get_controller, + ): + mock_get_model.return_value = juju.model.Model() + mock_get_application.return_value = FakeApplication() + result = self.loop.run_until_complete( + self.libjuju.check_application_exists( + "model", + "app", + ) + ) + self.assertEqual(result, True) + + mock_get_application.assert_called_once() + mock_get_controller.assert_called_once() + mock_get_model.assert_called_once() + mock_disconnect_controller.assert_called_once() + mock_disconnect_model.assert_called_once() + + def test_no_application( + self, + mock_get_application, + mock_disconnect_controller, + mock_disconnect_model, + mock_get_model, + mock_get_controller, + ): + mock_get_model.return_value = juju.model.Model() + mock_get_application.return_value = None + result = self.loop.run_until_complete( + self.libjuju.check_application_exists( + "model", + "app", + ) + ) + self.assertEqual(result, False) + + mock_get_application.assert_called_once() + mock_get_controller.assert_called_once() + mock_get_model.assert_called_once() + mock_disconnect_controller.assert_called_once() + mock_disconnect_model.assert_called_once() + + +@asynctest.mock.patch("n2vc.libjuju.Libjuju.get_controller") +@asynctest.mock.patch("n2vc.libjuju.Libjuju.get_model") +@asynctest.mock.patch("n2vc.libjuju.Libjuju.disconnect_model") +@asynctest.mock.patch("n2vc.libjuju.Libjuju.disconnect_controller") +@asynctest.mock.patch("n2vc.libjuju.Libjuju._get_application") +@asynctest.mock.patch("n2vc.libjuju.Libjuju._get_machine_info") +class AddUnitTest(LibjujuTestCase): + def setUp(self): + super(AddUnitTest, self).setUp() + + @asynctest.mock.patch("n2vc.juju_watcher.JujuModelWatcher.wait_for") + @asynctest.mock.patch("asyncio.sleep") + def test_successful( + self, + mock_sleep, + mock_wait_for, + mock_get_machine_info, + mock_get_application, + mock_disconnect_controller, + mock_disconnect_model, + mock_get_model, + mock_get_controller, + ): + mock_get_model.return_value = juju.model.Model() + mock_get_application.return_value = FakeApplication() + mock_get_machine_info.return_value = FakeMachine(), "series" + self.loop.run_until_complete( + self.libjuju.add_unit( + "existing_app", + "model", + "machine", + ) + ) + + mock_wait_for.assert_called_once() + mock_get_application.assert_called_once() + mock_get_controller.assert_called_once() + mock_get_model.assert_called_once() + mock_disconnect_controller.assert_called_once() + mock_disconnect_model.assert_called_once() + + def test_no_app( + self, + mock_get_machine_info, + mock_get_application, + mock_disconnect_controller, + mock_disconnect_model, + mock_get_model, + mock_get_controller, + ): + mock_get_model.return_value = juju.model.Model() + mock_get_application.return_value = None + with self.assertRaises(JujuApplicationNotFound): + self.loop.run_until_complete( + self.libjuju.add_unit( + "existing_app", + "model", + "machine", + ) + ) + + mock_get_application.assert_called_once() + mock_get_controller.assert_called_once() + mock_get_model.assert_called_once() + mock_disconnect_controller.assert_called_once() + mock_disconnect_model.assert_called_once() + + def test_no_machine( + self, + mock_get_machine_info, + mock_get_application, + mock_disconnect_controller, + mock_disconnect_model, + mock_get_model, + mock_get_controller, + ): + mock_get_model.return_value = juju.model.Model() + mock_get_application.return_value = FakeApplication() + mock_get_machine_info.side_effect = JujuMachineNotFound() + with self.assertRaises(JujuMachineNotFound): + self.loop.run_until_complete( + self.libjuju.add_unit( + "existing_app", + "model", + "machine", + ) + ) + + mock_get_application.assert_called_once() + mock_get_controller.assert_called_once() + mock_get_model.assert_called_once() + mock_disconnect_controller.assert_called_once() + mock_disconnect_model.assert_called_once() + + +@asynctest.mock.patch("n2vc.libjuju.Libjuju.get_controller") +@asynctest.mock.patch("n2vc.libjuju.Libjuju.get_model") +@asynctest.mock.patch("n2vc.libjuju.Libjuju.disconnect_model") +@asynctest.mock.patch("n2vc.libjuju.Libjuju.disconnect_controller") +@asynctest.mock.patch("n2vc.libjuju.Libjuju._get_application") +@asynctest.mock.patch("n2vc.libjuju.Libjuju._get_unit") +class DestroyUnitTest(LibjujuTestCase): + def setUp(self): + super(DestroyUnitTest, self).setUp() + + @asynctest.mock.patch("asyncio.sleep") + def test_successful( + self, + mock_sleep, + mock_get_unit, + mock_get_application, + mock_disconnect_controller, + mock_disconnect_model, + mock_get_model, + mock_get_controller, + ): + mock_get_model.return_value = juju.model.Model() + mock_get_application.return_value = FakeApplication() + + self.loop.run_until_complete( + self.libjuju.destroy_unit("app", "model", "machine", 0) + ) + + mock_get_unit.assert_called() + mock_get_application.assert_called_once() + mock_get_controller.assert_called_once() + mock_get_model.assert_called_once() + mock_disconnect_controller.assert_called_once() + mock_disconnect_model.assert_called_once() + + def test_no_app( + self, + mock_get_unit, + mock_get_application, + mock_disconnect_controller, + mock_disconnect_model, + mock_get_model, + mock_get_controller, + ): + mock_get_model.return_value = juju.model.Model() + mock_get_application.return_value = None + + with self.assertRaises(JujuApplicationNotFound): + self.loop.run_until_complete( + self.libjuju.destroy_unit("app", "model", "machine") + ) + + mock_get_application.assert_called_once() + mock_get_controller.assert_called_once() + mock_get_model.assert_called_once() + mock_disconnect_controller.assert_called_once() + mock_disconnect_model.assert_called_once() + + def test_no_unit( + self, + mock_get_unit, + mock_get_application, + mock_disconnect_controller, + mock_disconnect_model, + mock_get_model, + mock_get_controller, + ): + mock_get_model.return_value = juju.model.Model() + mock_get_application.return_value = FakeApplication() + mock_get_unit.return_value = None + + with self.assertRaises(JujuError): + self.loop.run_until_complete( + self.libjuju.destroy_unit("app", "model", "machine") + ) + + mock_get_unit.assert_called_once() + mock_get_application.assert_called_once() + mock_get_controller.assert_called_once() + mock_get_model.assert_called_once() + mock_disconnect_controller.assert_called_once() + mock_disconnect_model.assert_called_once() diff --git a/n2vc/tests/unit/utils.py b/n2vc/tests/unit/utils.py index 7e8907f..b2d5c60 100644 --- a/n2vc/tests/unit/utils.py +++ b/n2vc/tests/unit/utils.py @@ -123,6 +123,7 @@ class FakeManualMachine(MagicMock): model_name = "FAKE MODEL" entity_type = "machine" safe_data = {"instance-id": "manual:myid"} + series = "FAKE SERIES" async def destroy(self, force): pass @@ -162,6 +163,12 @@ class FakeUnit(MagicMock): async def run_action(self, action_name, **kwargs): return FakeAction() + @property + def machine_id(self): + return "existing_machine_id" + + name = "existing_unit" + class FakeApplication(AsyncMock): async def set_config(self, config): @@ -170,6 +177,9 @@ class FakeApplication(AsyncMock): async def add_unit(self, to): pass + async def destroy_unit(self, unit_name): + pass + async def get_actions(self): return ["existing_action"]