12 # FIXME: this should load the juju inside or modules without having to
13 # explicitly install it. Check why it's not working.
14 # Load our subtree of the juju library
15 path
= os
.path
.abspath(os
.path
.join(os
.path
.dirname(__file__
), '..'))
16 path
= os
.path
.join(path
, "modules/libjuju/")
17 if path
not in sys
.path
:
18 sys
.path
.insert(1, path
)
20 from juju
.controller
import Controller
21 from juju
.model
import ModelObserver
24 # We might need this to connect to the websocket securely, but test and verify.
26 ssl
._create
_default
_https
_context
= ssl
._create
_unverified
_context
27 except AttributeError:
28 # Legacy Python doesn't verify by default (see pep-0476)
29 # https://www.python.org/dev/peps/pep-0476/
34 class JujuCharmNotFound(Exception):
35 """The Charm can't be found or is not readable."""
38 class JujuApplicationExists(Exception):
39 """The Application already exists."""
42 class N2VCPrimitiveExecutionFailed(Exception):
43 """Something failed while attempting to execute a primitive."""
46 # Quiet the debug logging
47 logging
.getLogger('websockets.protocol').setLevel(logging
.INFO
)
48 logging
.getLogger('juju.client.connection').setLevel(logging
.WARN
)
49 logging
.getLogger('juju.model').setLevel(logging
.WARN
)
50 logging
.getLogger('juju.machine').setLevel(logging
.WARN
)
53 class VCAMonitor(ModelObserver
):
54 """Monitor state changes within the Juju Model."""
59 def __init__(self
, ns_name
):
60 self
.log
= logging
.getLogger(__name__
)
62 self
.ns_name
= ns_name
64 def AddApplication(self
, application_name
, callback
, *callback_args
):
65 if application_name
not in self
.applications
:
66 self
.applications
[application_name
] = {
68 'callback_args': callback_args
71 def RemoveApplication(self
, application_name
):
72 if application_name
in self
.applications
:
73 del self
.applications
[application_name
]
75 async def on_change(self
, delta
, old
, new
, model
):
76 """React to changes in the Juju model."""
78 if delta
.entity
== "unit":
79 # Ignore change events from other applications
80 if delta
.data
['application'] not in self
.applications
.keys():
85 application_name
= delta
.data
['application']
87 callback
= self
.applications
[application_name
]['callback']
89 self
.applications
[application_name
]['callback_args']
92 # Fire off a callback with the application state
96 delta
.data
['application'],
98 new
.workload_status_message
,
102 # This is a charm being removed
106 delta
.data
['application'],
110 except Exception as e
:
111 self
.log
.debug("[1] notify_callback exception: {}".format(e
))
113 elif delta
.entity
== "action":
114 # TODO: Decide how we want to notify the user of actions
116 # uuid = delta.data['id'] # The Action's unique id
117 # msg = delta.data['message'] # The output of the action
119 # if delta.data['status'] == "pending":
120 # # The action is queued
122 # elif delta.data['status'] == "completed""
123 # # The action was successful
125 # elif delta.data['status'] == "failed":
126 # # The action failed.
134 # Create unique models per network service
135 # Document all public functions
150 :param vcaconfig dict A dictionary containing the VCA configuration
152 :param artifacts str The directory where charms required by a vnfd are
156 n2vc = N2VC(vcaconfig={
157 'secret': 'MzI3MDJhOTYxYmM0YzRjNTJiYmY1Yzdm',
159 'ip-address': '10.44.127.137',
161 'artifacts': '/path/to/charms'
165 # Initialize instance-level variables
168 self
.controller
= None
169 self
.connecting
= False
170 self
.authenticated
= False
179 self
.default_model
= None
193 self
.log
= logging
.getLogger(__name__
)
195 # Quiet websocket traffic
196 logging
.getLogger('websockets.protocol').setLevel(logging
.INFO
)
197 logging
.getLogger('juju.client.connection').setLevel(logging
.WARN
)
198 logging
.getLogger('model').setLevel(logging
.WARN
)
199 # logging.getLogger('websockets.protocol').setLevel(logging.DEBUG)
201 self
.log
.debug('JujuApi: instantiated')
207 if user
.startswith('user-'):
210 self
.user
= 'user-{}'.format(user
)
212 self
.endpoint
= '%s:%d' % (server
, int(port
))
214 self
.artifacts
= artifacts
216 self
.loop
= loop
or asyncio
.get_event_loop()
219 """Close any open connections."""
222 def notify_callback(self
, model_name
, application_name
, status
, message
,
223 callback
=None, *callback_args
):
232 except Exception as e
:
233 self
.log
.error("[0] notify_callback exception {}".format(e
))
238 async def CreateNetworkService(self
, nsd
):
239 """Create a new model to encapsulate this network service.
241 Create a new model in the Juju controller to encapsulate the
242 charms associated with a network service.
244 You can pass either the nsd record or the id of the network
245 service, but this method will fail without one of them.
247 if not self
.authenticated
:
250 # Ideally, we will create a unique model per network service.
251 # This change will require all components, i.e., LCM and SO, to use
252 # N2VC for 100% compatibility. If we adopt unique models for the LCM,
253 # services deployed via LCM would't be manageable via SO and vice versa
255 return self
.default_model
257 async def DeployCharms(self
, model_name
, application_name
, vnfd
,
258 charm_path
, params
={}, machine_spec
={},
259 callback
=None, *callback_args
):
260 """Deploy one or more charms associated with a VNF.
262 Deploy the charm(s) referenced in a VNF Descriptor.
264 :param str model_name: The name of the network service.
265 :param str application_name: The name of the application
266 :param dict vnfd: The name of the application
267 :param str charm_path: The path to the Juju charm
268 :param dict params: A dictionary of runtime parameters
271 'rw_mgmt_ip': '1.2.3.4',
272 # Pass the initial-config-primitives section of the vnf or vdu
273 'initial-config-primitives': {...}
275 :param dict machine_spec: A dictionary describing the machine to
279 'hostname': '1.2.3.4',
280 'username': 'ubuntu',
282 :param obj callback: A callback function to receive status changes.
283 :param tuple callback_args: A list of arguments to be passed to the
287 ########################################################
288 # Verify the path to the charm exists and is readable. #
289 ########################################################
290 if not os
.path
.exists(charm_path
):
291 self
.log
.debug("Charm path doesn't exist: {}".format(charm_path
))
292 self
.notify_callback(
299 raise JujuCharmNotFound("No artifacts configured.")
301 ################################
302 # Login to the Juju controller #
303 ################################
304 if not self
.authenticated
:
305 self
.log
.debug("Authenticating with Juju")
308 ##########################################
309 # Get the model for this network service #
310 ##########################################
311 # TODO: In a point release, we will use a model per deployed network
312 # service. In the meantime, we will always use the 'default' model.
313 model_name
= 'default'
314 model
= await self
.get_model(model_name
)
316 ########################################
317 # Verify the application doesn't exist #
318 ########################################
319 app
= await self
.get_application(model
, application_name
)
321 raise JujuApplicationExists("Can't deploy application \"{}\" to model \"{}\" because it already exists.".format(application_name
, model_name
))
323 ################################################################
324 # Register this application with the model-level event monitor #
325 ################################################################
327 self
.monitors
[model_name
].AddApplication(
333 ########################################################
334 # Check for specific machine placement (native charms) #
335 ########################################################
337 if machine_spec
.keys():
338 if all(k
in machine_spec
for k
in ['host', 'user']):
339 # Enlist an existing machine as a Juju unit
340 machine
= await model
.add_machine(spec
='ssh:{}@{}:{}'.format(
341 machine_spec
['user'],
342 machine_spec
['host'],
343 self
.GetPrivateKeyPath(),
347 #######################################
348 # Get the initial charm configuration #
349 #######################################
352 if 'rw_mgmt_ip' in params
:
353 rw_mgmt_ip
= params
['rw_mgmt_ip']
355 if 'initial-config-primitive' not in params
:
356 params
['initial-config-primitive'] = {}
358 initial_config
= self
._get
_config
_from
_dict
(
359 params
['initial-config-primitive'],
360 {'<rw_mgmt_ip>': rw_mgmt_ip
}
363 self
.log
.debug("JujuApi: Deploying charm ({}) from {}".format(
369 ########################################################
370 # Deploy the charm and apply the initial configuration #
371 ########################################################
372 app
= await model
.deploy(
373 # We expect charm_path to be either the path to the charm on disk
374 # or in the format of cs:series/name
376 # This is the formatted, unique name for this charm
377 application_name
=application_name
,
378 # Proxy charms should use the current LTS. This will need to be
379 # changed for native charms.
381 # Apply the initial 'config' primitive during deployment
382 config
=initial_config
,
383 # Where to deploy the charm to.
387 # #######################################
388 # # Execute initial config primitive(s) #
389 # #######################################
390 await self
.ExecuteInitialPrimitives(
398 # # Build a sequential list of the primitives to execute
399 # for primitive in params['initial-config-primitive']:
401 # if primitive['name'] == 'config':
402 # # This is applied when the Application is deployed
405 # seq = primitive['seq']
408 # if 'parameter' in primitive:
409 # params = primitive['parameter']
411 # primitives[seq] = {
412 # 'name': primitive['name'],
413 # 'parameters': self._map_primitive_parameters(
415 # {'<rw_mgmt_ip>': rw_mgmt_ip}
419 # for primitive in sorted(primitives):
420 # await self.ExecutePrimitive(
423 # primitives[primitive]['name'],
426 # **primitives[primitive]['parameters'],
428 # except N2VCPrimitiveExecutionFailed as e:
430 # "[N2VC] Exception executing primitive: {}".format(e)
434 async def GetPrimitiveStatus(self
, model_name
, uuid
):
435 """Get the status of an executed Primitive.
437 The status of an executed Primitive will be one of three values:
444 if not self
.authenticated
:
447 # FIXME: This is hard-coded until model-per-ns is added
448 model_name
= 'default'
450 model
= await self
.get_model(model_name
)
452 results
= await model
.get_action_status(uuid
)
455 status
= results
[uuid
]
457 except Exception as e
:
459 "Caught exception while getting primitive status: {}".format(e
)
461 raise N2VCPrimitiveExecutionFailed(e
)
465 async def GetPrimitiveOutput(self
, model_name
, uuid
):
466 """Get the output of an executed Primitive.
468 Note: this only returns output for a successfully executed primitive.
472 if not self
.authenticated
:
475 # FIXME: This is hard-coded until model-per-ns is added
476 model_name
= 'default'
478 model
= await self
.get_model(model_name
)
479 results
= await model
.get_action_output(uuid
, 60)
480 except Exception as e
:
482 "Caught exception while getting primitive status: {}".format(e
)
484 raise N2VCPrimitiveExecutionFailed(e
)
488 # async def ProvisionMachine(self, model_name, hostname, username):
489 # """Provision machine for usage with Juju.
491 # Provisions a previously instantiated machine for use with Juju.
494 # if not self.authenticated:
497 # # FIXME: This is hard-coded until model-per-ns is added
498 # model_name = 'default'
500 # model = await self.get_model(model_name)
501 # model.add_machine(spec={})
503 # machine = await model.add_machine(spec='ssh:{}@{}:{}'.format(
510 # except Exception as e:
512 # "Caught exception while getting primitive status: {}".format(e)
514 # raise N2VCPrimitiveExecutionFailed(e)
516 def GetPrivateKeyPath(self
):
517 homedir
= os
.environ
['HOME']
518 sshdir
= "{}/.ssh".format(homedir
)
519 private_key_path
= "{}/id_n2vc_rsa".format(sshdir
)
520 return private_key_path
522 async def GetPublicKey(self
):
523 """Get the N2VC SSH public key.abs
525 Returns the SSH public key, to be injected into virtual machines to
526 be managed by the VCA.
528 The first time this is run, a ssh keypair will be created. The public
529 key is injected into a VM so that we can provision the machine with
530 Juju, after which Juju will communicate with the VM directly via the
535 # Find the path to where we expect our key to live.
536 homedir
= os
.environ
['HOME']
537 sshdir
= "{}/.ssh".format(homedir
)
538 if not os
.path
.exists(sshdir
):
541 private_key_path
= "{}/id_n2vc_rsa".format(sshdir
)
542 public_key_path
= "{}.pub".format(private_key_path
)
544 # If we don't have a key generated, generate it.
545 if not os
.path
.exists(private_key_path
):
546 cmd
= "ssh-keygen -t {} -b {} -N '' -f {}".format(
551 subprocess
.check_output(shlex
.split(cmd
))
553 # Read the public key
554 with
open(public_key_path
, "r") as f
:
555 public_key
= f
.readline()
559 async def ExecuteInitialPrimitives(self
, model_name
, application_name
,
560 params
, callback
=None, *callback_args
):
561 """Execute multiple primitives.
563 Execute multiple primitives as declared in initial-config-primitive.
564 This is useful in cases where the primitives initially failed -- for
565 example, if the charm is a proxy but the proxy hasn't been configured
571 # Build a sequential list of the primitives to execute
572 for primitive
in params
['initial-config-primitive']:
574 if primitive
['name'] == 'config':
577 seq
= primitive
['seq']
580 if 'parameter' in primitive
:
581 params
= primitive
['parameter']
584 'name': primitive
['name'],
585 'parameters': self
._map
_primitive
_parameters
(
587 {'<rw_mgmt_ip>': None}
591 for primitive
in sorted(primitives
):
593 await self
.ExecutePrimitive(
596 primitives
[primitive
]['name'],
599 **primitives
[primitive
]['parameters'],
602 except N2VCPrimitiveExecutionFailed
as e
:
604 "[N2VC] Exception executing primitive: {}".format(e
)
609 async def ExecutePrimitive(self
, model_name
, application_name
, primitive
,
610 callback
, *callback_args
, **params
):
611 """Execute a primitive of a charm for Day 1 or Day 2 configuration.
613 Execute a primitive defined in the VNF descriptor.
615 :param str model_name: The name of the network service.
616 :param str application_name: The name of the application
617 :param str primitive: The name of the primitive to execute.
618 :param obj callback: A callback function to receive status changes.
619 :param tuple callback_args: A list of arguments to be passed to the
621 :param dict params: A dictionary of key=value pairs representing the
622 primitive's parameters
625 'rw_mgmt_ip': '1.2.3.4',
626 # Pass the initial-config-primitives section of the vnf or vdu
627 'initial-config-primitives': {...}
630 self
.log
.debug("Executing {}".format(primitive
))
633 if not self
.authenticated
:
636 # FIXME: This is hard-coded until model-per-ns is added
637 model_name
= 'default'
639 model
= await self
.get_model(model_name
)
641 if primitive
== 'config':
642 # config is special, and expecting params to be a dictionary
643 await self
.set_config(
649 app
= await self
.get_application(model
, application_name
)
651 # Run against the first (and probably only) unit in the app
654 action
= await unit
.run_action(primitive
, **params
)
656 except Exception as e
:
658 "Caught exception while executing primitive: {}".format(e
)
660 raise N2VCPrimitiveExecutionFailed(e
)
663 async def RemoveCharms(self
, model_name
, application_name
, callback
=None,
665 """Remove a charm from the VCA.
667 Remove a charm referenced in a VNF Descriptor.
669 :param str model_name: The name of the network service.
670 :param str application_name: The name of the application
671 :param obj callback: A callback function to receive status changes.
672 :param tuple callback_args: A list of arguments to be passed to the
676 if not self
.authenticated
:
679 model
= await self
.get_model(model_name
)
680 app
= await self
.get_application(model
, application_name
)
682 # Remove this application from event monitoring
683 self
.monitors
[model_name
].RemoveApplication(application_name
)
685 # self.notify_callback(model_name, application_name, "removing", callback, *callback_args)
687 "Removing the application {}".format(application_name
)
691 # Notify the callback that this charm has been removed.
692 self
.notify_callback(
700 except Exception as e
:
701 print("Caught exception: {}".format(e
))
705 async def DestroyNetworkService(self
, nsd
):
706 raise NotImplementedError()
708 async def GetMetrics(self
, model_name
, application_name
):
709 """Get the metrics collected by the VCA.
711 :param model_name The name of the model
712 :param application_name The name of the application
715 model
= await self
.get_model(model_name
)
716 app
= await self
.get_application(model
, application_name
)
718 metrics
= await app
.get_metrics()
722 async def HasApplication(self
, model_name
, application_name
):
723 model
= await self
.get_model(model_name
)
724 app
= await self
.get_application(model
, application_name
)
730 async def add_relation(self
, a
, b
, via
=None):
732 Add a relation between two application endpoints.
734 :param a An application endpoint
735 :param b An application endpoint
736 :param via The egress subnet(s) for outbound traffic, e.g.,
737 (192.168.0.0/16,10.0.0.0/8)
739 if not self
.authenticated
:
742 m
= await self
.get_model()
744 m
.add_relation(a
, b
, via
)
748 # async def apply_config(self, config, application):
749 # """Apply a configuration to the application."""
750 # print("JujuApi: Applying configuration to {}.".format(
753 # return await self.set_config(application=application, config=config)
755 def _get_config_from_dict(self
, config_primitive
, values
):
756 """Transform the yang config primitive to dict.
765 for primitive
in config_primitive
:
766 if primitive
['name'] == 'config':
767 # config = self._map_primitive_parameters()
768 for parameter
in primitive
['parameter']:
769 param
= str(parameter
['name'])
770 if parameter
['value'] == "<rw_mgmt_ip>":
771 config
[param
] = str(values
[parameter
['value']])
773 config
[param
] = str(parameter
['value'])
777 def _map_primitive_parameters(self
, parameters
, values
):
779 for parameter
in parameters
:
780 param
= str(parameter
['name'])
782 # Typecast parameter value, if present
783 if 'data-type' in parameter
:
784 paramtype
= str(parameter
['data-type']).lower()
787 if paramtype
== "integer":
788 value
= int(parameter
['value'])
789 elif paramtype
== "boolean":
790 value
= bool(parameter
['value'])
792 value
= str(parameter
['value'])
794 if parameter
['value'] == "<rw_mgmt_ip>":
795 params
[param
] = str(values
[parameter
['value']])
797 params
[param
] = value
800 def _get_config_from_yang(self
, config_primitive
, values
):
801 """Transform the yang config primitive to dict."""
803 for primitive
in config_primitive
.values():
804 if primitive
['name'] == 'config':
805 for parameter
in primitive
['parameter'].values():
806 param
= str(parameter
['name'])
807 if parameter
['value'] == "<rw_mgmt_ip>":
808 config
[param
] = str(values
[parameter
['value']])
810 config
[param
] = str(parameter
['value'])
815 def FormatApplicationName(self
, *args
):
817 Generate a Juju-compatible Application name
819 :param args tuple: Positional arguments to be used to construct the
823 - Only accepts characters a-z and non-consequitive dashes (-)
824 - Application name should not exceed 50 characters
828 FormatApplicationName("ping_pong_ns", "ping_vnf", "a")
832 for c
in "-".join(list(args
)):
835 elif not c
.isalpha():
838 return re
.sub('\-+', '-', appname
.lower())
840 # def format_application_name(self, nsd_name, vnfr_name, member_vnf_index=0):
841 # """Format the name of the application
844 # - Only accepts characters a-z and non-consequitive dashes (-)
845 # - Application name should not exceed 50 characters
847 # name = "{}-{}-{}".format(nsd_name, vnfr_name, member_vnf_index)
851 # c = chr(97 + int(c))
852 # elif not c.isalpha():
855 # return re.sub('\-+', '-', new_name.lower())
857 def format_model_name(self
, name
):
858 """Format the name of model.
860 Model names may only contain lowercase letters, digits and hyphens
863 return name
.replace('_', '-').lower()
865 async def get_application(self
, model
, application
):
866 """Get the deployed application."""
867 if not self
.authenticated
:
871 if application
and model
:
872 if model
.applications
:
873 if application
in model
.applications
:
874 app
= model
.applications
[application
]
878 async def get_model(self
, model_name
='default'):
879 """Get a model from the Juju Controller.
881 Note: Model objects returned must call disconnected() before it goes
883 if not self
.authenticated
:
886 if model_name
not in self
.models
:
887 self
.models
[model_name
] = await self
.controller
.get_model(
890 self
.refcount
['model'] += 1
892 # Create an observer for this model
893 self
.monitors
[model_name
] = VCAMonitor(model_name
)
894 self
.models
[model_name
].add_observer(self
.monitors
[model_name
])
896 return self
.models
[model_name
]
898 async def login(self
):
899 """Login to the Juju controller."""
901 if self
.authenticated
:
904 self
.connecting
= True
906 self
.log
.debug("JujuApi: Logging into controller")
909 self
.controller
= Controller(loop
=self
.loop
)
913 "Connecting to controller... ws://{}:{} as {}/{}".format(
920 await self
.controller
.connect(
921 endpoint
=self
.endpoint
,
923 password
=self
.secret
,
926 self
.refcount
['controller'] += 1
928 # current_controller no longer exists
929 # self.log.debug("Connecting to current controller...")
930 # await self.controller.connect_current()
931 # await self.controller.connect(
932 # endpoint=self.endpoint,
933 # username=self.user,
936 self
.log
.fatal("VCA credentials not configured.")
938 self
.authenticated
= True
939 self
.log
.debug("JujuApi: Logged into controller")
941 async def logout(self
):
942 """Logout of the Juju controller."""
943 if not self
.authenticated
:
947 if self
.default_model
:
948 self
.log
.debug("Disconnecting model {}".format(
951 await self
.default_model
.disconnect()
952 self
.refcount
['model'] -= 1
953 self
.default_model
= None
955 for model
in self
.models
:
956 await self
.models
[model
].disconnect()
957 self
.refcount
['model'] -= 1
958 self
.models
[model
] = None
961 self
.log
.debug("Disconnecting controller {}".format(
964 await self
.controller
.disconnect()
965 self
.refcount
['controller'] -= 1
966 self
.controller
= None
968 self
.authenticated
= False
970 self
.log
.debug(self
.refcount
)
972 except Exception as e
:
974 "Fatal error logging out of Juju Controller: {}".format(e
)
978 # async def remove_application(self, name):
979 # """Remove the application."""
980 # if not self.authenticated:
983 # app = await self.get_application(name)
985 # self.log.debug("JujuApi: Destroying application {}".format(
989 # await app.destroy()
991 async def remove_relation(self
, a
, b
):
993 Remove a relation between two application endpoints
995 :param a An application endpoint
996 :param b An application endpoint
998 if not self
.authenticated
:
1001 m
= await self
.get_model()
1003 m
.remove_relation(a
, b
)
1005 await m
.disconnect()
1007 async def resolve_error(self
, application
=None):
1008 """Resolve units in error state."""
1009 if not self
.authenticated
:
1012 app
= await self
.get_application(self
.default_model
, application
)
1015 "JujuApi: Resolving errors for application {}".format(
1020 for unit
in app
.units
:
1021 app
.resolved(retry
=True)
1023 async def run_action(self
, application
, action_name
, **params
):
1024 """Execute an action and return an Action object."""
1025 if not self
.authenticated
:
1034 app
= await self
.get_application(self
.default_model
, application
)
1036 # We currently only have one unit per application
1037 # so use the first unit available.
1041 "JujuApi: Running Action {} against Application {}".format(
1047 action
= await unit
.run_action(action_name
, **params
)
1049 # Wait for the action to complete
1052 result
['status'] = action
.status
1053 result
['action']['tag'] = action
.data
['id']
1054 result
['action']['results'] = action
.results
1058 async def set_config(self
, model_name
, application
, config
):
1059 """Apply a configuration to the application."""
1060 if not self
.authenticated
:
1063 app
= await self
.get_application(model_name
, application
)
1065 self
.log
.debug("JujuApi: Setting config for Application {}".format(
1068 await app
.set_config(config
)
1070 # Verify the config is set
1071 newconf
= await app
.get_config()
1073 if config
[key
] != newconf
[key
]['value']:
1074 self
.log
.debug("JujuApi: Config not set! Key {} Value {} doesn't match {}".format(key
, config
[key
], newconf
[key
]))
1076 # async def set_parameter(self, parameter, value, application=None):
1077 # """Set a config parameter for a service."""
1078 # if not self.authenticated:
1079 # await self.login()
1081 # self.log.debug("JujuApi: Setting {}={} for Application {}".format(
1086 # return await self.apply_config(
1087 # {parameter: value},
1088 # application=application,
1091 async def wait_for_application(self
, model_name
, application_name
,
1093 """Wait for an application to become active."""
1094 if not self
.authenticated
:
1097 # TODO: In a point release, we will use a model per deployed network
1098 # service. In the meantime, we will always use the 'default' model.
1099 model_name
= 'default'
1100 model
= await self
.get_model(model_name
)
1102 app
= await self
.get_application(model
, application_name
)
1103 self
.log
.debug("Application: {}".format(app
))
1104 # app = await self.get_application(model_name, application_name)
1107 "JujuApi: Waiting {} seconds for Application {}".format(
1113 await model
.block_until(
1115 unit
.agent_status
== 'idle' and unit
.workload_status
in
1116 ['active', 'unknown'] for unit
in app
.units