Fix bug #502
[osm/N2VC.git] / n2vc / vnf.py
1
2 import logging
3 import os
4 import os.path
5 import re
6 import ssl
7 import sys
8 import time
9
10 # FIXME: this should load the juju inside or modules without having to
11 # explicitly install it. Check why it's not working.
12 # Load our subtree of the juju library
13 path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
14 path = os.path.join(path, "modules/libjuju/")
15 if path not in sys.path:
16 sys.path.insert(1, path)
17
18 from juju.controller import Controller
19 from juju.model import Model, ModelObserver
20
21
22 # We might need this to connect to the websocket securely, but test and verify.
23 try:
24 ssl._create_default_https_context = ssl._create_unverified_context
25 except AttributeError:
26 # Legacy Python doesn't verify by default (see pep-0476)
27 # https://www.python.org/dev/peps/pep-0476/
28 pass
29
30
31 # Custom exceptions
32 class JujuCharmNotFound(Exception):
33 """The Charm can't be found or is not readable."""
34
35
36 class JujuApplicationExists(Exception):
37 """The Application already exists."""
38
39
40 class N2VCPrimitiveExecutionFailed(Exception):
41 """Something failed while attempting to execute a primitive."""
42
43
44 # Quiet the debug logging
45 logging.getLogger('websockets.protocol').setLevel(logging.INFO)
46 logging.getLogger('juju.client.connection').setLevel(logging.WARN)
47 logging.getLogger('juju.model').setLevel(logging.WARN)
48 logging.getLogger('juju.machine').setLevel(logging.WARN)
49
50
51 class VCAMonitor(ModelObserver):
52 """Monitor state changes within the Juju Model."""
53 log = None
54 ns_name = None
55 applications = {}
56
57 def __init__(self, ns_name):
58 self.log = logging.getLogger(__name__)
59
60 self.ns_name = ns_name
61
62 def AddApplication(self, application_name, callback, *callback_args):
63 if application_name not in self.applications:
64 self.applications[application_name] = {
65 'callback': callback,
66 'callback_args': callback_args
67 }
68
69 def RemoveApplication(self, application_name):
70 if application_name in self.applications:
71 del self.applications[application_name]
72
73 async def on_change(self, delta, old, new, model):
74 """React to changes in the Juju model."""
75
76 if delta.entity == "unit":
77 # Ignore change events from other applications
78 if delta.data['application'] not in self.applications.keys():
79 return
80
81 try:
82
83 application_name = delta.data['application']
84
85 callback = self.applications[application_name]['callback']
86 callback_args = self.applications[application_name]['callback_args']
87
88 if old and new:
89 old_status = old.workload_status
90 new_status = new.workload_status
91
92 if old_status == new_status:
93 """The workload status may fluctuate around certain events,
94 so wait until the status has stabilized before triggering
95 the callback."""
96 if callback:
97 callback(
98 self.ns_name,
99 delta.data['application'],
100 new_status,
101 new.workload_status_message,
102 *callback_args)
103
104 if old and not new:
105 # This is a charm being removed
106 if callback:
107 callback(
108 self.ns_name,
109 delta.data['application'],
110 "removed",
111 "",
112 *callback_args)
113 except Exception as e:
114 self.log.debug("[1] notify_callback exception {}".format(e))
115 elif delta.entity == "action":
116 # TODO: Decide how we want to notify the user of actions
117
118 # uuid = delta.data['id'] # The Action's unique id
119 # msg = delta.data['message'] # The output of the action
120 #
121 # if delta.data['status'] == "pending":
122 # # The action is queued
123 # pass
124 # elif delta.data['status'] == "completed""
125 # # The action was successful
126 # pass
127 # elif delta.data['status'] == "failed":
128 # # The action failed.
129 # pass
130
131 pass
132
133 ########
134 # TODO
135 #
136 # Create unique models per network service
137 # Document all public functions
138
139
140 class N2VC:
141
142 # Juju API
143 api = None
144 log = None
145 controller = None
146 connecting = False
147 authenticated = False
148
149 models = {}
150 default_model = None
151
152 # Model Observers
153 monitors = {}
154
155 # VCA config
156 hostname = ""
157 port = 17070
158 username = ""
159 secret = ""
160
161 def __init__(self,
162 log=None,
163 server='127.0.0.1',
164 port=17070,
165 user='admin',
166 secret=None,
167 artifacts=None
168 ):
169 """Initialize N2VC
170
171 :param vcaconfig dict A dictionary containing the VCA configuration
172
173 :param artifacts str The directory where charms required by a vnfd are
174 stored.
175
176 :Example:
177 n2vc = N2VC(vcaconfig={
178 'secret': 'MzI3MDJhOTYxYmM0YzRjNTJiYmY1Yzdm',
179 'user': 'admin',
180 'ip-address': '10.44.127.137',
181 'port': 17070,
182 'artifacts': '/path/to/charms'
183 })
184
185 """
186
187 if log:
188 self.log = log
189 else:
190 self.log = logging.getLogger(__name__)
191
192 # Quiet websocket traffic
193 logging.getLogger('websockets.protocol').setLevel(logging.INFO)
194 logging.getLogger('juju.client.connection').setLevel(logging.WARN)
195 logging.getLogger('model').setLevel(logging.WARN)
196 # logging.getLogger('websockets.protocol').setLevel(logging.DEBUG)
197
198 self.log.debug('JujuApi: instantiated')
199
200 self.server = server
201 self.port = port
202
203 self.secret = secret
204 if user.startswith('user-'):
205 self.user = user
206 else:
207 self.user = 'user-{}'.format(user)
208
209 self.endpoint = '%s:%d' % (server, int(port))
210
211 self.artifacts = artifacts
212
213 def __del__(self):
214 """Close any open connections."""
215 yield self.logout()
216
217 def notify_callback(self, model_name, application_name, status, message, callback=None, *callback_args):
218 try:
219 if callback:
220 callback(model_name, application_name, status, message, *callback_args)
221 except Exception as e:
222 self.log.error("[0] notify_callback exception {}".format(e))
223 raise e
224 return True
225
226 # Public methods
227 async def CreateNetworkService(self, nsd):
228 """Create a new model to encapsulate this network service.
229
230 Create a new model in the Juju controller to encapsulate the
231 charms associated with a network service.
232
233 You can pass either the nsd record or the id of the network
234 service, but this method will fail without one of them.
235 """
236 if not self.authenticated:
237 await self.login()
238
239 # Ideally, we will create a unique model per network service.
240 # This change will require all components, i.e., LCM and SO, to use
241 # N2VC for 100% compatibility. If we adopt unique models for the LCM,
242 # services deployed via LCM would't be manageable via SO and vice versa
243
244 return self.default_model
245
246 async def DeployCharms(self, model_name, application_name, vnfd, charm_path, params={}, machine_spec={}, callback=None, *callback_args):
247 """Deploy one or more charms associated with a VNF.
248
249 Deploy the charm(s) referenced in a VNF Descriptor.
250
251 :param str model_name: The name of the network service.
252 :param str application_name: The name of the application
253 :param dict vnfd: The name of the application
254 :param str charm_path: The path to the Juju charm
255 :param dict params: A dictionary of runtime parameters
256 Examples::
257 {
258 'rw_mgmt_ip': '1.2.3.4',
259 # Pass the initial-config-primitives section of the vnf or vdu
260 'initial-config-primitives': {...}
261 }
262 :param dict machine_spec: A dictionary describing the machine to install to
263 Examples::
264 {
265 'hostname': '1.2.3.4',
266 'username': 'ubuntu',
267 }
268 :param obj callback: A callback function to receive status changes.
269 :param tuple callback_args: A list of arguments to be passed to the callback
270 """
271
272 ########################################################
273 # Verify the path to the charm exists and is readable. #
274 ########################################################
275 if not os.path.exists(charm_path):
276 self.log.debug("Charm path doesn't exist: {}".format(charm_path))
277 self.notify_callback(model_name, application_name, "failed", callback, *callback_args)
278 raise JujuCharmNotFound("No artifacts configured.")
279
280 ################################
281 # Login to the Juju controller #
282 ################################
283 if not self.authenticated:
284 self.log.debug("Authenticating with Juju")
285 await self.login()
286
287 ##########################################
288 # Get the model for this network service #
289 ##########################################
290 # TODO: In a point release, we will use a model per deployed network
291 # service. In the meantime, we will always use the 'default' model.
292 model_name = 'default'
293 model = await self.get_model(model_name)
294
295 ########################################
296 # Verify the application doesn't exist #
297 ########################################
298 app = await self.get_application(model, application_name)
299 if app:
300 raise JujuApplicationExists("Can't deploy application \"{}\" to model \"{}\" because it already exists.".format(application_name, model_name))
301
302 ################################################################
303 # Register this application with the model-level event monitor #
304 ################################################################
305 if callback:
306 self.monitors[model_name].AddApplication(
307 application_name,
308 callback,
309 *callback_args
310 )
311
312 ########################################################
313 # Check for specific machine placement (native charms) #
314 ########################################################
315 to = ""
316 if machine_spec.keys():
317 # TODO: This needs to be tested.
318 # if all(k in machine_spec for k in ['hostname', 'username']):
319 # # Enlist the existing machine in Juju
320 # machine = await self.model.add_machine(spec='ssh:%@%'.format(
321 # specs['host'],
322 # specs['user'],
323 # ))
324 # to = machine.id
325 pass
326
327 #######################################
328 # Get the initial charm configuration #
329 #######################################
330
331 rw_mgmt_ip = None
332 if 'rw_mgmt_ip' in params:
333 rw_mgmt_ip = params['rw_mgmt_ip']
334
335 initial_config = self._get_config_from_dict(
336 params['initial-config-primitive'],
337 {'<rw_mgmt_ip>': rw_mgmt_ip}
338 )
339
340 self.log.debug("JujuApi: Deploying charm ({}) from {}".format(
341 application_name,
342 charm_path,
343 to=to,
344 ))
345
346 ########################################################
347 # Deploy the charm and apply the initial configuration #
348 ########################################################
349 app = await model.deploy(
350 # We expect charm_path to be either the path to the charm on disk
351 # or in the format of cs:series/name
352 charm_path,
353 # This is the formatted, unique name for this charm
354 application_name=application_name,
355 # Proxy charms should use the current LTS. This will need to be
356 # changed for native charms.
357 series='xenial',
358 # Apply the initial 'config' primitive during deployment
359 config=initial_config,
360 # TBD: Where to deploy the charm to.
361 to=None,
362 )
363
364 # #######################################
365 # # Execute initial config primitive(s) #
366 # #######################################
367 primitives = {}
368
369 # Build a sequential list of the primitives to execute
370 for primitive in params['initial-config-primitive']:
371 try:
372 if primitive['name'] == 'config':
373 # This is applied when the Application is deployed
374 pass
375 else:
376 seq = primitive['seq']
377
378 params = {}
379 if 'parameter' in primitive:
380 params = primitive['parameter']
381
382 primitives[seq] = {
383 'name': primitive['name'],
384 'parameters': self._map_primitive_parameters(
385 params,
386 {'<rw_mgmt_ip>': rw_mgmt_ip}
387 ),
388 }
389
390 for primitive in sorted(primitives):
391 await self.ExecutePrimitive(
392 model_name,
393 application_name,
394 primitives[primitive]['name'],
395 callback,
396 callback_args,
397 **primitives[primitive]['parameters'],
398 )
399 except N2VCPrimitiveExecutionFailed as e:
400 self.log.debug(
401 "[N2VC] Exception executing primitive: {}".format(e)
402 )
403 raise
404
405 async def ExecutePrimitive(self, model_name, application_name, primitive, callback, *callback_args, **params):
406 """Execute a primitive of a charm for Day 1 or Day 2 configuration.
407
408 Execute a primitive defined in the VNF descriptor.
409
410 :param str model_name: The name of the network service.
411 :param str application_name: The name of the application
412 :param str primitive: The name of the primitive to execute.
413 :param obj callback: A callback function to receive status changes.
414 :param tuple callback_args: A list of arguments to be passed to the callback function.
415 :param dict params: A dictionary of key=value pairs representing the primitive's parameters
416 Examples::
417 {
418 'rw_mgmt_ip': '1.2.3.4',
419 # Pass the initial-config-primitives section of the vnf or vdu
420 'initial-config-primitives': {...}
421 }
422 """
423 uuid = None
424 try:
425 if not self.authenticated:
426 await self.login()
427
428 # FIXME: This is hard-coded until model-per-ns is added
429 model_name = 'default'
430
431 model = await self.controller.get_model(model_name)
432
433 if primitive == 'config':
434 # config is special, and expecting params to be a dictionary
435 self.log.debug("Setting charm configuration for {}".format(application_name))
436 self.log.debug(params['params'])
437 await self.set_config(model, application_name, params['params'])
438 else:
439 app = await self.get_application(model, application_name)
440 if app:
441 # Run against the first (and probably only) unit in the app
442 unit = app.units[0]
443 if unit:
444 self.log.debug("Executing primitive {}".format(primitive))
445 action = await unit.run_action(primitive, **params)
446 uuid = action.id
447 await model.disconnect()
448 except Exception as e:
449 self.log.debug("Caught exception while executing primitive: {}".format(e))
450 raise N2VCPrimitiveExecutionFailed(e)
451 return uuid
452
453 async def RemoveCharms(self, model_name, application_name, callback=None, *callback_args):
454 """Remove a charm from the VCA.
455
456 Remove a charm referenced in a VNF Descriptor.
457
458 :param str model_name: The name of the network service.
459 :param str application_name: The name of the application
460 :param obj callback: A callback function to receive status changes.
461 :param tuple callback_args: A list of arguments to be passed to the callback function.
462 """
463 try:
464 if not self.authenticated:
465 await self.login()
466
467 model = await self.get_model(model_name)
468 app = await self.get_application(model, application_name)
469 if app:
470 # Remove this application from event monitoring
471 self.monitors[model_name].RemoveApplication(application_name)
472
473 # self.notify_callback(model_name, application_name, "removing", callback, *callback_args)
474 self.log.debug("Removing the application {}".format(application_name))
475 await app.remove()
476
477 # Notify the callback that this charm has been removed.
478 self.notify_callback(model_name, application_name, "removed", callback, *callback_args)
479
480 except Exception as e:
481 print("Caught exception: {}".format(e))
482 self.log.debug(e)
483 raise e
484
485 async def DestroyNetworkService(self, nsd):
486 raise NotImplementedError()
487
488 async def GetMetrics(self, model_name, application_name):
489 """Get the metrics collected by the VCA.
490
491 :param model_name The name of the model
492 :param application_name The name of the application
493 """
494 metrics = {}
495 model = await self.get_model(model_name)
496 app = await self.get_application(model, application_name)
497 if app:
498 metrics = await app.get_metrics()
499
500 return metrics
501
502 # Non-public methods
503 async def add_relation(self, a, b, via=None):
504 """
505 Add a relation between two application endpoints.
506
507 :param a An application endpoint
508 :param b An application endpoint
509 :param via The egress subnet(s) for outbound traffic, e.g.,
510 (192.168.0.0/16,10.0.0.0/8)
511 """
512 if not self.authenticated:
513 await self.login()
514
515 m = await self.get_model()
516 try:
517 m.add_relation(a, b, via)
518 finally:
519 await m.disconnect()
520
521 # async def apply_config(self, config, application):
522 # """Apply a configuration to the application."""
523 # print("JujuApi: Applying configuration to {}.".format(
524 # application
525 # ))
526 # return await self.set_config(application=application, config=config)
527
528 def _get_config_from_dict(self, config_primitive, values):
529 """Transform the yang config primitive to dict.
530
531 Expected result:
532
533 config = {
534 'config':
535 }
536 """
537 config = {}
538 for primitive in config_primitive:
539 if primitive['name'] == 'config':
540 # config = self._map_primitive_parameters()
541 for parameter in primitive['parameter']:
542 param = str(parameter['name'])
543 if parameter['value'] == "<rw_mgmt_ip>":
544 config[param] = str(values[parameter['value']])
545 else:
546 config[param] = str(parameter['value'])
547
548 return config
549
550 def _map_primitive_parameters(self, parameters, values):
551 params = {}
552 for parameter in parameters:
553 param = str(parameter['name'])
554 if parameter['value'] == "<rw_mgmt_ip>":
555 params[param] = str(values[parameter['value']])
556 else:
557 """
558 The Juju API uses strictly typed data-types, so we must make
559 sure the parameters from the VNFD match the appropriate type.
560
561 The honus will still be on the operator, to make sure the
562 data-type in the VNFD matches the one in the charm. N2VC will
563 raise N2VCPrimitiveExecutionFailed when there is a mismatch.
564
565 There are three data types supported by the YANG model:
566 # - STRING
567 # - INTEGER
568 # - BOOLEAN
569
570 Each parameter will look like this:
571 {
572 'seq': '3',
573 'name': 'testint',
574 'parameter': [
575 {
576 'name': 'interval',
577 'data-type': 'INTEGER',
578 'value': 20
579 }
580 ]
581 }
582 """
583
584 if 'value' in parameter:
585 # String is the default format
586 val = str(parameter['value'])
587
588 # If the data-type is explicitly set, cast to that type.
589 if 'data-type' in parameter:
590 dt = parameter['data-type'].upper()
591 if dt == "INTEGER":
592 val = int(val)
593
594 elif dt == "BOOLEAN":
595 if val in ['true', 'false', '0', '1']:
596 val = True
597 else:
598 val = False
599
600 params[param] = val
601 return params
602
603 def _get_config_from_yang(self, config_primitive, values):
604 """Transform the yang config primitive to dict."""
605 config = {}
606 for primitive in config_primitive.values():
607 if primitive['name'] == 'config':
608 for parameter in primitive['parameter'].values():
609 param = str(parameter['name'])
610 if parameter['value'] == "<rw_mgmt_ip>":
611 config[param] = str(values[parameter['value']])
612 else:
613 config[param] = str(parameter['value'])
614
615 return config
616
617 def FormatApplicationName(self, *args):
618 """
619 Generate a Juju-compatible Application name
620
621 :param args tuple: Positional arguments to be used to construct the
622 application name.
623
624 Limitations::
625 - Only accepts characters a-z and non-consequitive dashes (-)
626 - Application name should not exceed 50 characters
627
628 Examples::
629
630 FormatApplicationName("ping_pong_ns", "ping_vnf", "a")
631 """
632
633 appname = ""
634 for c in "-".join(list(args)):
635 if c.isdigit():
636 c = chr(97 + int(c))
637 elif not c.isalpha():
638 c = "-"
639 appname += c
640 return re.sub('\-+', '-', appname.lower())
641
642
643 # def format_application_name(self, nsd_name, vnfr_name, member_vnf_index=0):
644 # """Format the name of the application
645 #
646 # Limitations:
647 # - Only accepts characters a-z and non-consequitive dashes (-)
648 # - Application name should not exceed 50 characters
649 # """
650 # name = "{}-{}-{}".format(nsd_name, vnfr_name, member_vnf_index)
651 # new_name = ''
652 # for c in name:
653 # if c.isdigit():
654 # c = chr(97 + int(c))
655 # elif not c.isalpha():
656 # c = "-"
657 # new_name += c
658 # return re.sub('\-+', '-', new_name.lower())
659
660 def format_model_name(self, name):
661 """Format the name of model.
662
663 Model names may only contain lowercase letters, digits and hyphens
664 """
665
666 return name.replace('_', '-').lower()
667
668 async def get_application(self, model, application):
669 """Get the deployed application."""
670 if not self.authenticated:
671 await self.login()
672
673 app = None
674 if application and model:
675 if model.applications:
676 if application in model.applications:
677 app = model.applications[application]
678
679 return app
680
681 async def get_model(self, model_name='default'):
682 """Get a model from the Juju Controller.
683
684 Note: Model objects returned must call disconnected() before it goes
685 out of scope."""
686 if not self.authenticated:
687 await self.login()
688
689 if model_name not in self.models:
690 print("connecting to model {}".format(model_name))
691 self.models[model_name] = await self.controller.get_model(model_name)
692
693 # Create an observer for this model
694 self.monitors[model_name] = VCAMonitor(model_name)
695 self.models[model_name].add_observer(self.monitors[model_name])
696
697 return self.models[model_name]
698
699 async def login(self):
700 """Login to the Juju controller."""
701
702 if self.authenticated:
703 return
704
705 self.connecting = True
706
707 self.log.debug("JujuApi: Logging into controller")
708
709 cacert = None
710 self.controller = Controller()
711
712 if self.secret:
713 self.log.debug("Connecting to controller... ws://{}:{} as {}/{}".format(self.endpoint, self.port, self.user, self.secret))
714 await self.controller.connect(
715 endpoint=self.endpoint,
716 username=self.user,
717 password=self.secret,
718 cacert=cacert,
719 )
720 else:
721 # current_controller no longer exists
722 # self.log.debug("Connecting to current controller...")
723 # await self.controller.connect_current()
724 # await self.controller.connect(
725 # endpoint=self.endpoint,
726 # username=self.user,
727 # cacert=cacert,
728 # )
729 self.log.fatal("VCA credentials not configured.")
730
731 self.authenticated = True
732 self.log.debug("JujuApi: Logged into controller")
733
734 # self.default_model = await self.controller.get_model("default")
735
736 async def logout(self):
737 """Logout of the Juju controller."""
738 if not self.authenticated:
739 return
740
741 try:
742 if self.default_model:
743 self.log.debug("Disconnecting model {}".format(self.default_model))
744 await self.default_model.disconnect()
745 self.default_model = None
746
747 for model in self.models:
748 await self.models[model].disconnect()
749
750 if self.controller:
751 self.log.debug("Disconnecting controller {}".format(self.controller))
752 await self.controller.disconnect()
753 # self.controller = None
754
755 self.authenticated = False
756 except Exception as e:
757 self.log.fail("Fatal error logging out of Juju Controller: {}".format(e))
758 raise e
759
760
761 # async def remove_application(self, name):
762 # """Remove the application."""
763 # if not self.authenticated:
764 # await self.login()
765 #
766 # app = await self.get_application(name)
767 # if app:
768 # self.log.debug("JujuApi: Destroying application {}".format(
769 # name,
770 # ))
771 #
772 # await app.destroy()
773
774 async def remove_relation(self, a, b):
775 """
776 Remove a relation between two application endpoints
777
778 :param a An application endpoint
779 :param b An application endpoint
780 """
781 if not self.authenticated:
782 await self.login()
783
784 m = await self.get_model()
785 try:
786 m.remove_relation(a, b)
787 finally:
788 await m.disconnect()
789
790 async def resolve_error(self, application=None):
791 """Resolve units in error state."""
792 if not self.authenticated:
793 await self.login()
794
795 app = await self.get_application(self.default_model, application)
796 if app:
797 self.log.debug("JujuApi: Resolving errors for application {}".format(
798 application,
799 ))
800
801 for unit in app.units:
802 app.resolved(retry=True)
803
804 async def run_action(self, application, action_name, **params):
805 """Execute an action and return an Action object."""
806 if not self.authenticated:
807 await self.login()
808 result = {
809 'status': '',
810 'action': {
811 'tag': None,
812 'results': None,
813 }
814 }
815 app = await self.get_application(self.default_model, application)
816 if app:
817 # We currently only have one unit per application
818 # so use the first unit available.
819 unit = app.units[0]
820
821 self.log.debug("JujuApi: Running Action {} against Application {}".format(
822 action_name,
823 application,
824 ))
825
826 action = await unit.run_action(action_name, **params)
827
828 # Wait for the action to complete
829 await action.wait()
830
831 result['status'] = action.status
832 result['action']['tag'] = action.data['id']
833 result['action']['results'] = action.results
834
835 return result
836
837 async def set_config(self, model_name, application, config):
838 """Apply a configuration to the application."""
839 if not self.authenticated:
840 await self.login()
841
842 app = await self.get_application(model_name, application)
843 if app:
844 self.log.debug("JujuApi: Setting config for Application {}".format(
845 application,
846 ))
847 await app.set_config(config)
848
849 # Verify the config is set
850 newconf = await app.get_config()
851 for key in config:
852 if config[key] != newconf[key]['value']:
853 self.log.debug("JujuApi: Config not set! Key {} Value {} doesn't match {}".format(key, config[key], newconf[key]))
854
855 # async def set_parameter(self, parameter, value, application=None):
856 # """Set a config parameter for a service."""
857 # if not self.authenticated:
858 # await self.login()
859 #
860 # self.log.debug("JujuApi: Setting {}={} for Application {}".format(
861 # parameter,
862 # value,
863 # application,
864 # ))
865 # return await self.apply_config(
866 # {parameter: value},
867 # application=application,
868 # )
869
870 async def wait_for_application(self, name, timeout=300):
871 """Wait for an application to become active."""
872 if not self.authenticated:
873 await self.login()
874
875 app = await self.get_application(self.default_model, name)
876 if app:
877 self.log.debug(
878 "JujuApi: Waiting {} seconds for Application {}".format(
879 timeout,
880 name,
881 )
882 )
883
884 await self.default_model.block_until(
885 lambda: all(
886 unit.agent_status == 'idle'
887 and unit.workload_status
888 in ['active', 'unknown'] for unit in app.units
889 ),
890 timeout=timeout
891 )