From 1a15d1c84fc826fa7996c1c9d221a324edd33432 Mon Sep 17 00:00:00 2001 From: Adam Israel Date: Mon, 23 Oct 2017 12:00:49 -0400 Subject: [PATCH 1/1] Update libjuju - fix licensing quirks - refresh against libjuju master Signed-off-by: Adam Israel --- examples/credential.py | 46 ++++ juju/user.py | 80 +++++++ modules/libjuju/.travis.yml | 2 - modules/libjuju/VERSION | 2 +- .../docs/_extensions/automembersummary.py | 20 +- modules/libjuju/docs/changelog.rst | 11 + modules/libjuju/juju/application.py | 74 +++++- modules/libjuju/juju/client/_client.py | 23 +- modules/libjuju/juju/client/client.py | 2 +- modules/libjuju/juju/client/connection.py | 34 ++- modules/libjuju/juju/client/facade.py | 84 ++++--- modules/libjuju/juju/client/overrides.py | 45 ++++ modules/libjuju/juju/client/runner.py | 19 +- modules/libjuju/juju/controller.py | 212 +++++++++++++----- modules/libjuju/juju/errors.py | 13 +- modules/libjuju/juju/loop.py | 2 +- modules/libjuju/juju/model.py | 11 +- modules/libjuju/juju/utils.py | 10 +- modules/libjuju/tests/base.py | 31 ++- .../tests/integration/test_application.py | 30 +++ .../tests/integration/test_controller.py | 145 +++++++++--- .../libjuju/tests/integration/test_machine.py | 3 +- .../libjuju/tests/integration/test_model.py | 12 - modules/libjuju/tests/unit/test_client.py | 11 +- modules/libjuju/tox.ini | 18 +- 25 files changed, 737 insertions(+), 203 deletions(-) create mode 100644 examples/credential.py create mode 100644 juju/user.py diff --git a/examples/credential.py b/examples/credential.py new file mode 100644 index 0000000..f335af9 --- /dev/null +++ b/examples/credential.py @@ -0,0 +1,46 @@ +import sys +from juju import loop +from juju.controller import Controller + + +async def main(cloud_name, credential_name): + controller = Controller() + model = None + print('Connecting to controller') + await controller.connect_current() + try: + print('Adding model') + model = await controller.add_model( + 'test', + cloud_name=cloud_name, + credential_name=credential_name) + + # verify credential + print("Verify model's credential: {}".format( + model.info.cloud_credential_tag)) + + # verify we can deploy + print('Deploying ubuntu') + app = await model.deploy('ubuntu-10') + + print('Waiting for active') + await model.block_until( + lambda: app.units and all(unit.workload_status == 'active' + for unit in app.units)) + + print('Removing ubuntu') + await app.remove() + finally: + print('Cleaning up') + if model: + print('Removing model') + model_uuid = model.info.uuid + await model.disconnect() + await controller.destroy_model(model_uuid) + print('Disconnecting') + await controller.disconnect() + + +if __name__ == '__main__': + assert len(sys.argv) > 2, 'Please provide a cloud and credential name' + loop.run(main(sys.argv[1], sys.argv[2])) diff --git a/juju/user.py b/juju/user.py new file mode 100644 index 0000000..b8890e1 --- /dev/null +++ b/juju/user.py @@ -0,0 +1,80 @@ +import logging +from dateutil.parser import parse as parse_date + +from . import tag + +log = logging.getLogger(__name__) + + +class User(object): + def __init__(self, controller, user_info): + self.controller = controller + self._user_info = user_info + + @property + def tag(self): + return tag.user(self.username) + + @property + def username(self): + return self._user_info.username + + @property + def display_name(self): + return self._user_info.display_name + + @property + def last_connection(self): + return parse_date(self._user_info.last_connection) + + @property + def access(self): + return self._user_info.access + + @property + def date_created(self): + return self._user_info.date_created + + @property + def enabled(self): + return not self._user_info.disabled + + @property + def disabled(self): + return self._user_info.disabled + + @property + def created_by(self): + return self._user_info.created_by + + async def set_password(self, password): + """Update this user's password. + """ + await self.controller.change_user_password(self.username, password) + self._user_info.password = password + + async def grant(self, acl='login'): + """Set access level of this user on the controller. + + :param str acl: Access control ('login', 'add-model', or 'superuser') + """ + await self.controller.grant(self.username, acl) + self._user_info.access = acl + + async def revoke(self): + """Removes all access rights for this user from the controller. + """ + await self.controller.revoke(self.username) + self._user_info.access = '' + + async def disable(self): + """Disable this user. + """ + await self.controller.disable_user(self.username) + self._user_info.disabled = True + + async def enable(self): + """Re-enable this user. + """ + await self.controller.enable_user(self.username) + self._user_info.disabled = False diff --git a/modules/libjuju/.travis.yml b/modules/libjuju/.travis.yml index 0a57c38..16e88dc 100644 --- a/modules/libjuju/.travis.yml +++ b/modules/libjuju/.travis.yml @@ -19,5 +19,3 @@ script: - sudo ln -s /snap/bin/juju /usr/bin/juju || true - sudo -E sudo -u $USER -E bash -c "/snap/bin/juju bootstrap localhost test" - tox -e py35,integration - - sudo -E sudo -u $USER -E bash -c "/snap/bin/juju destroy-controller --destroy-all-models -y test" - - sudo snap remove juju diff --git a/modules/libjuju/VERSION b/modules/libjuju/VERSION index a918a2a..ee6cdce 100644 --- a/modules/libjuju/VERSION +++ b/modules/libjuju/VERSION @@ -1 +1 @@ -0.6.0 +0.6.1 diff --git a/modules/libjuju/docs/_extensions/automembersummary.py b/modules/libjuju/docs/_extensions/automembersummary.py index 898da62..cfe0b84 100644 --- a/modules/libjuju/docs/_extensions/automembersummary.py +++ b/modules/libjuju/docs/_extensions/automembersummary.py @@ -1,18 +1,16 @@ # Copyright 2014-2015 Canonical Limited. # -# This file is part of charm-helpers. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# charm-helpers is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 as -# published by the Free Software Foundation. +# http://www.apache.org/licenses/LICENSE-2.0 # -# charm-helpers is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with charm-helpers. If not, see . +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import importlib diff --git a/modules/libjuju/docs/changelog.rst b/modules/libjuju/docs/changelog.rst index 9da0cdc..d3d2e91 100644 --- a/modules/libjuju/docs/changelog.rst +++ b/modules/libjuju/docs/changelog.rst @@ -1,6 +1,17 @@ Change Log ---------- +0.6.1 +^^^^^ +Fri Sept 29 2017 + +* Fix failure when controller supports newer facade version (#145) +* Fix test failures (#163) +* Fix SSH key handling when adding a new model (#161) +* Make Application.upgrade_charm upgrade resources (#158) +* Expand integration tests to use stable/edge versions of juju (#155) +* Move docs to ReadTheDocs (https://pythonlibjuju.readthedocs.io/en/latest/) + 0.6.0 ^^^^^ Thu June 29 2017 diff --git a/modules/libjuju/juju/application.py b/modules/libjuju/juju/application.py index 8719a62..620e9c9 100644 --- a/modules/libjuju/juju/application.py +++ b/modules/libjuju/juju/application.py @@ -1,3 +1,17 @@ +# Copyright 2016 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import asyncio import logging @@ -342,8 +356,13 @@ class Application(model.ModelEntity): raise ValueError("switch and revision are mutually exclusive") client_facade = client.ClientFacade.from_connection(self.connection) + resources_facade = client.ResourcesFacade.from_connection( + self.connection) app_facade = client.ApplicationFacade.from_connection(self.connection) + charmstore = self.model.charmstore + charmstore_entity = None + if switch is not None: charm_url = switch if not charm_url.startswith('cs:'): @@ -354,18 +373,65 @@ class Application(model.ModelEntity): if revision is not None: charm_url = "%s-%d" % (charm_url, revision) else: - charmstore = self.model.charmstore - entity = await charmstore.entity(charm_url, channel=channel) - charm_url = entity['Id'] + charmstore_entity = await charmstore.entity(charm_url, + channel=channel) + charm_url = charmstore_entity['Id'] if charm_url == self.data['charm-url']: raise JujuError('already running charm "%s"' % charm_url) + # Update charm await client_facade.AddCharm( url=charm_url, channel=channel ) + # Update resources + if not charmstore_entity: + charmstore_entity = await charmstore.entity(charm_url, + channel=channel) + store_resources = charmstore_entity['Meta']['resources'] + + request_data = [client.Entity(self.tag)] + response = await resources_facade.ListResources(request_data) + existing_resources = { + resource.name: resource + for resource in response.results[0].resources + } + + resources_to_update = [ + resource for resource in store_resources + if resource['Name'] not in existing_resources or + existing_resources[resource['Name']].origin != 'upload' + ] + + if resources_to_update: + request_data = [ + client.CharmResource( + description=resource.get('Description'), + fingerprint=resource['Fingerprint'], + name=resource['Name'], + path=resource['Path'], + revision=resource['Revision'], + size=resource['Size'], + type_=resource['Type'], + origin='store', + ) for resource in resources_to_update + ] + response = await resources_facade.AddPendingResources( + self.tag, + charm_url, + request_data + ) + pending_ids = response.pending_ids + resource_ids = { + resource['Name']: id + for resource, id in zip(resources_to_update, pending_ids) + } + else: + resource_ids = None + + # Update application await app_facade.SetCharm( application=self.entity_id, channel=channel, @@ -374,7 +440,7 @@ class Application(model.ModelEntity): config_settings_yaml=None, force_series=force_series, force_units=force_units, - resource_ids=None, + resource_ids=resource_ids, storage_constraints=None ) diff --git a/modules/libjuju/juju/client/_client.py b/modules/libjuju/juju/client/_client.py index d510e11..2ef0ffd 100644 --- a/modules/libjuju/juju/client/_client.py +++ b/modules/libjuju/juju/client/_client.py @@ -1,7 +1,7 @@ # DO NOT CHANGE THIS FILE! This file is auto-generated by facade.py. # Changes will be overwritten/lost when the file is regenerated. -from juju.client._definitions import * +from juju.client._definitions import * # noqa from juju.client import _client1, _client2, _client3, _client4, _client5 @@ -15,22 +15,21 @@ CLIENTS = { } - def lookup_facade(name, version): """ Given a facade name and version, attempt to pull that facade out of the correct client.py file. """ - try: - facade = getattr(CLIENTS[str(version)], name) - except KeyError: - raise ImportError("No facades found for version {}".format(version)) - except AttributeError: - raise ImportError( - "No facade with name '{}' in version {}".format(name, version)) - return facade - + for _version in range(int(version), 0, -1): + try: + facade = getattr(CLIENTS[str(_version)], name) + return facade + except (KeyError, AttributeError): + continue + else: + raise ImportError("No supported version for facade: " + "{}".format(name)) class TypeFactory: @@ -363,5 +362,3 @@ class UserManagerFacade(TypeFactory): class VolumeAttachmentsWatcherFacade(TypeFactory): pass - - diff --git a/modules/libjuju/juju/client/client.py b/modules/libjuju/juju/client/client.py index 89b5248..2f3e49d 100644 --- a/modules/libjuju/juju/client/client.py +++ b/modules/libjuju/juju/client/client.py @@ -5,7 +5,7 @@ from . import _client, _definitions, overrides for o in overrides.__all__: - if not "Facade" in o: + if "Facade" not in o: # Override stuff in _definitions, which is all imported # into _client. We Monkey patch both the original class and # the ref in _client (import shenanigans are fun!) diff --git a/modules/libjuju/juju/client/connection.py b/modules/libjuju/juju/client/connection.py index 7457391..c09468c 100644 --- a/modules/libjuju/juju/client/connection.py +++ b/modules/libjuju/juju/client/connection.py @@ -413,7 +413,7 @@ class Connection: endpoints.extend(new_endpoints) else: # ran out of endpoints without a successful login - raise Exception("Couldn't authenticate to {}".format( + raise JujuConnectionError("Couldn't authenticate to {}".format( self._endpoint)) response = result['response'] @@ -584,6 +584,38 @@ class JujuData: def accounts(self): return self._load_yaml('accounts.yaml', 'controllers') + def credentials(self): + return self._load_yaml('credentials.yaml', 'credentials') + + def load_credential(self, cloud, name=None): + """Load a local credential. + + :param str cloud: Name of cloud to load credentials from. + :param str name: Name of credential. If None, the default credential + will be used, if available. + :returns: A CloudCredential instance, or None. + """ + try: + cloud = tag.untag('cloud-', cloud) + creds_data = self.credentials()[cloud] + if not name: + default_credential = creds_data.pop('default-credential', None) + default_region = creds_data.pop('default-region', None) # noqa + if default_credential: + name = creds_data['default-credential'] + elif len(creds_data) == 1: + name = list(creds_data)[0] + else: + return None, None + cred_data = creds_data[name] + auth_type = cred_data.pop('auth-type') + return name, client.CloudCredential( + auth_type=auth_type, + attrs=cred_data, + ) + except (KeyError, FileNotFoundError): + return None, None + def _load_yaml(self, filename, key): filepath = os.path.join(self.path, filename) with io.open(filepath, 'rt') as f: diff --git a/modules/libjuju/juju/client/facade.py b/modules/libjuju/juju/client/facade.py index c959e01..c015c5f 100644 --- a/modules/libjuju/juju/client/facade.py +++ b/modules/libjuju/juju/client/facade.py @@ -44,19 +44,19 @@ HEADER = """\ # Classes and helper functions that we'll write to _client.py LOOKUP_FACADE = ''' def lookup_facade(name, version): - """ - Given a facade name and version, attempt to pull that facade out - of the correct client.py file. + """ + Given a facade name and version, attempt to pull that facade out + of the correct client.py file. - """ - try: - facade = getattr(CLIENTS[str(version)], name) - except KeyError: - raise ImportError("No facades found for version {}".format(version)) - except AttributeError: - raise ImportError( - "No facade with name '{}' in version {}".format(name, version)) - return facade + """ + try: + facade = getattr(CLIENTS[str(version)], name) + except KeyError: + raise ImportError("No facades found for version {}".format(version)) + except AttributeError: + raise ImportError( + "No facade with name '{}' in version {}".format(name, version)) + return facade ''' @@ -127,6 +127,7 @@ class TypeRegistry(dict): return self[refname] + _types = TypeRegistry() _registry = KindRegistry() CLASSES = {} @@ -257,7 +258,7 @@ def buildTypes(schema, capture): for kind in sorted((k for k in _types if not isinstance(k, str)), key=lambda x: str(x)): name = _types[kind] - if name in capture and not name in NAUGHTY_CLASSES: + if name in capture and name not in NAUGHTY_CLASSES: continue args = Args(kind) # Write Factory class for _client.py @@ -277,9 +278,7 @@ class {}(Type): pprint.pformat(args.SchemaToPyMapping(), width=999), ", " if args else "", args.as_kwargs(), - textwrap.indent(args.get_doc(), INDENT * 2)) - ] - assignments = args._get_arg_str(False, False) + textwrap.indent(args.get_doc(), INDENT * 2))] if not args: source.append("{}pass".format(INDENT * 2)) @@ -289,7 +288,9 @@ class {}(Type): arg_type = arg[1] arg_type_name = strcast(arg_type) if arg_type in basic_types: - source.append("{}self.{} = {}".format(INDENT * 2, arg_name, arg_name)) + source.append("{}self.{} = {}".format(INDENT * 2, + arg_name, + arg_name)) elif issubclass(arg_type, typing.Sequence): value_type = ( arg_type_name.__parameters__[0] @@ -297,10 +298,16 @@ class {}(Type): else None ) if type(value_type) is typing.TypeVar: - source.append("{}self.{} = [{}.from_json(o) for o in {} or []]".format( - INDENT * 2, arg_name, strcast(value_type), arg_name)) + source.append( + "{}self.{} = [{}.from_json(o) " + "for o in {} or []]".format(INDENT * 2, + arg_name, + strcast(value_type), + arg_name)) else: - source.append("{}self.{} = {}".format(INDENT * 2, arg_name, arg_name)) + source.append("{}self.{} = {}".format(INDENT * 2, + arg_name, + arg_name)) elif issubclass(arg_type, typing.Mapping): value_type = ( arg_type_name.__parameters__[1] @@ -308,15 +315,28 @@ class {}(Type): else None ) if type(value_type) is typing.TypeVar: - source.append("{}self.{} = {{k: {}.from_json(v) for k, v in ({} or dict()).items()}}".format( - INDENT * 2, arg_name, strcast(value_type), arg_name)) + source.append( + "{}self.{} = {{k: {}.from_json(v) " + "for k, v in ({} or dict()).items()}}".format( + INDENT * 2, + arg_name, + strcast(value_type), + arg_name)) else: - source.append("{}self.{} = {}".format(INDENT * 2, arg_name, arg_name)) + source.append("{}self.{} = {}".format(INDENT * 2, + arg_name, + arg_name)) elif type(arg_type) is typing.TypeVar: - source.append("{}self.{} = {}.from_json({}) if {} else None".format( - INDENT * 2, arg_name, arg_type_name, arg_name, arg_name)) + source.append("{}self.{} = {}.from_json({}) " + "if {} else None".format(INDENT * 2, + arg_name, + arg_type_name, + arg_name, + arg_name)) else: - source.append("{}self.{} = {}".format(INDENT * 2, arg_name, arg_name)) + source.append("{}self.{} = {}".format(INDENT * 2, + arg_name, + arg_name)) source = "\n".join(source) capture.clear(name) @@ -435,7 +455,10 @@ def makeFunc(cls, name, params, result, async=True): ''' # map input types to rpc msg _params = dict() - msg = dict(type='{cls.name}', request='{name}', version={cls.version}, params=_params) + msg = dict(type='{cls.name}', + request='{name}', + version={cls.version}, + params=_params) {assignments} reply = {await}self.rpc(msg) return reply @@ -539,7 +562,7 @@ class Type: return d def to_json(self): - return json.dumps(self.serialize()) + return json.dumps(self.serialize(), cls=TypeEncoder, sort_keys=True) class Schema(dict): @@ -576,7 +599,7 @@ class Schema(dict): if not defs: return for d, data in defs.items(): - if d in _registry and not d in NAUGHTY_CLASSES: + if d in _registry and d not in NAUGHTY_CLASSES: continue node = self.deref(data, d) kind = node.get("type") @@ -762,6 +785,7 @@ def generate_facades(options): return captures + def setup(): parser = argparse.ArgumentParser() parser.add_argument("-s", "--schema", default="juju/client/schemas*") @@ -769,6 +793,7 @@ def setup(): options = parser.parse_args() return options + def main(): options = setup() @@ -780,5 +805,6 @@ def main(): write_definitions(captures, options, last_version) write_client(captures, options) + if __name__ == '__main__': main() diff --git a/modules/libjuju/juju/client/overrides.py b/modules/libjuju/juju/client/overrides.py index f439adb..5e98e56 100644 --- a/modules/libjuju/juju/client/overrides.py +++ b/modules/libjuju/juju/client/overrides.py @@ -11,6 +11,7 @@ __all__ = [ 'Number', 'Binary', 'ConfigValue', + 'Resource', ] __patches__ = [ @@ -273,3 +274,47 @@ class ConfigValue(_definitions.ConfigValue): return '<{} source={} value={}>'.format(type(self).__name__, repr(self.source), repr(self.value)) + + +class Resource(Type): + _toSchema = {'application': 'application', + 'charmresource': 'CharmResource', + 'id_': 'id', + 'pending_id': 'pending-id', + 'timestamp': 'timestamp', + 'username': 'username', + 'name': 'name', + 'origin': 'origin'} + _toPy = {'CharmResource': 'charmresource', + 'application': 'application', + 'id': 'id_', + 'pending-id': 'pending_id', + 'timestamp': 'timestamp', + 'username': 'username', + 'name': 'name', + 'origin': 'origin'} + + def __init__(self, charmresource=None, application=None, id_=None, + pending_id=None, timestamp=None, username=None, name=None, + origin=None, **unknown_fields): + ''' + charmresource : CharmResource + application : str + id_ : str + pending_id : str + timestamp : str + username : str + name: str + origin : str + ''' + if charmresource: + self.charmresource = _client.CharmResource.from_json(charmresource) + else: + self.charmresource = None + self.application = application + self.id_ = id_ + self.pending_id = pending_id + self.timestamp = timestamp + self.username = username + self.name = name + self.origin = origin diff --git a/modules/libjuju/juju/client/runner.py b/modules/libjuju/juju/client/runner.py index 61f2963..6545bc4 100644 --- a/modules/libjuju/juju/client/runner.py +++ b/modules/libjuju/juju/client/runner.py @@ -1,6 +1,4 @@ - - class AsyncRunner: async def __call__(self, facade_method, *args, **kwargs): await self.connection.rpc(facade_method(*args, **kwargs)) @@ -15,14 +13,9 @@ class ThreadedRunner: # This could let us fake the protocol we want # while decoupling the protocol from the RPC and the IO/Process context -# The problem is leaking the runtime impl details to the top levels of the API with -# async def -# By handling the Marshal/Unmarshal side of RPC as a protocol we can leave the RPC running to a specific -# delegate without altering the method signatures. -# This still isn't quite right though as async is co-op multitasking and the methods still need to know -# not to block or they will pause other execution - - - - - +# The problem is leaking the runtime impl details to the top levels of the API +# with async def By handling the Marshal/Unmarshal side of RPC as a protocol we +# can leave the RPC running to a specific delegate without altering the method +# signatures. This still isn't quite right though as async is co-op +# multitasking and the methods still need to know not to block or they will +# pause other execution diff --git a/modules/libjuju/juju/controller.py b/modules/libjuju/juju/controller.py index 9b452c7..55ea55e 100644 --- a/modules/libjuju/juju/controller.py +++ b/modules/libjuju/juju/controller.py @@ -1,11 +1,13 @@ import asyncio import logging +from . import errors from . import tag from . import utils from .client import client from .client import connection from .model import Model +from .user import User log = logging.getLogger(__name__) @@ -39,9 +41,11 @@ class Controller(object): """Connect to the current Juju controller. """ - self.connection = ( - await connection.Connection.connect_current_controller( - max_frame_size=self.max_frame_size)) + jujudata = connection.JujuData() + controller_name = jujudata.current_controller() + if not controller_name: + raise errors.JujuConnectionError('No current controller') + return await self.connect_controller(controller_name) async def connect_controller(self, controller_name): """Connect to a Juju controller by name. @@ -61,6 +65,47 @@ class Controller(object): await self.connection.close() self.connection = None + async def add_credential(self, name=None, credential=None, cloud=None, + owner=None): + """Add or update a credential to the controller. + + :param str name: Name of new credential. If None, the default + local credential is used. Name must be provided if a credential + is given. + :param CloudCredential credential: Credential to add. If not given, + it will attempt to read from local data, if available. + :param str cloud: Name of cloud to associate the credential with. + Defaults to the same cloud as the controller. + :param str owner: Username that will own the credential. Defaults to + the current user. + :returns: Name of credential that was uploaded. + """ + if not cloud: + cloud = await self.get_cloud() + + if not owner: + owner = self.connection.info['user-info']['identity'] + + if credential and not name: + raise errors.JujuError('Name must be provided for credential') + + if not credential: + name, credential = connection.JujuData().load_credential(cloud, + name) + if credential is None: + raise errors.JujuError('Unable to find credential: ' + '{}'.format(name)) + + log.debug('Uploading credential %s', name) + cloud_facade = client.CloudFacade.from_connection(self.connection) + await cloud_facade.UpdateCredentials([ + client.UpdateCloudCredential( + tag=tag.credential(cloud, tag.untag('user-', owner), name), + credential=credential, + )]) + + return name + async def add_model( self, model_name, cloud_name=None, credential_name=None, owner=None, config=None, region=None): @@ -70,9 +115,8 @@ class Controller(object): :param str cloud_name: Name of the cloud in which to create the model, e.g. 'aws'. Defaults to same cloud as controller. :param str credential_name: Name of the credential to use when - creating the model. Defaults to current credential. If you - pass a credential_name, you must also pass a cloud_name, - even if it's the default cloud. + creating the model. If not given, it will attempt to find a + default credential. :param str owner: Username that will own the model. Defaults to the current user. :param dict config: Model configuration. @@ -85,6 +129,16 @@ class Controller(object): owner = owner or self.connection.info['user-info']['identity'] cloud_name = cloud_name or await self.get_cloud() + try: + # attempt to add/update the credential from local data if available + credential_name = await self.add_credential( + name=credential_name, + cloud=cloud_name, + owner=owner) + except errors.JujuError: + # if it's not available locally, assume it's on the controller + pass + if credential_name: credential = tag.credential( cloud_name, @@ -96,6 +150,11 @@ class Controller(object): log.debug('Creating model %s', model_name) + if not config or 'authorized-keys' not in config: + config = config or {} + config['authorized-keys'] = await utils.read_ssh_key( + loop=self.loop) + model_info = await model_facade.CreateModel( tag.cloud(cloud_name), config, @@ -105,24 +164,6 @@ class Controller(object): region ) - # Add our ssh key to the model, to work around - # https://bugs.launchpad.net/juju/+bug/1643076 - try: - ssh_key = await utils.read_ssh_key(loop=self.loop) - - if self.controller_name: - model_name = "{}:{}".format(self.controller_name, model_name) - - cmd = ['juju', 'add-ssh-key', '-m', model_name, ssh_key] - - await utils.execute_process(*cmd, log=log, loop=self.loop) - except Exception: - log.exception( - "Could not add ssh key to model. You will not be able " - "to ssh into machines in this model. " - "Manually running `juju add-ssh-key ` in the cli " - "may fix this problem.") - model = Model() await model.connect( self.connection.endpoint, @@ -136,24 +177,28 @@ class Controller(object): return model - async def destroy_models(self, *uuids): + async def destroy_models(self, *models): """Destroy one or more models. - :param str \*uuids: UUIDs of models to destroy + :param str \*models: Names or UUIDs of models to destroy """ + uuids = await self._model_uuids() + models = [uuids[model] if model in uuids else model + for model in models] + model_facade = client.ModelManagerFacade.from_connection( self.connection) log.debug( 'Destroying model%s %s', - '' if len(uuids) == 1 else 's', - ', '.join(uuids) + '' if len(models) == 1 else 's', + ', '.join(models) ) await model_facade.DestroyModels([ - client.Entity(tag.model(uuid)) - for uuid in uuids + client.Entity(tag.model(model)) + for model in models ]) destroy_model = destroy_models @@ -161,18 +206,26 @@ class Controller(object): """Add a user to this controller. :param str username: Username + :param str password: Password :param str display_name: Display name - :param str acl: Access control, e.g. 'read' - :param list models: Models to which the user is granted access - + :returns: A :class:`~juju.user.User` instance """ if not display_name: display_name = username user_facade = client.UserManagerFacade.from_connection(self.connection) - users = [{'display_name': display_name, - 'password': password, - 'username': username}] - return await user_facade.AddUser(users) + users = [client.AddUser(display_name=display_name, + username=username, + password=password)] + await user_facade.AddUser(users) + return await self.get_user(username) + + async def remove_user(self, username): + """Remove a user from this controller. + """ + client_facade = client.UserManagerFacade.from_connection( + self.connection) + user = tag.user(username) + await client_facade.RemoveUser([client.Entity(user)]) async def change_user_password(self, username, password): """Change the password for a user in this controller. @@ -231,17 +284,31 @@ class Controller(object): cloud = list(result.clouds.keys())[0] # only lives on one cloud return tag.untag('cloud-', cloud) - async def get_models(self, all_=False, username=None): - """Return list of available models on this controller. + async def _model_uuids(self, all_=False, username=None): + controller_facade = client.ControllerFacade.from_connection( + self.connection) + for attempt in (1, 2, 3): + try: + response = await controller_facade.AllModels() + return {um.model.name: um.model.uuid + for um in response.user_models} + except errors.JujuAPIError as e: + # retry concurrency error until resolved in Juju + # see: https://bugs.launchpad.net/juju/+bug/1721786 + if 'has been removed' not in e.message or attempt == 3: + raise + await asyncio.sleep(attempt, loop=self.loop) + + async def list_models(self, all_=False, username=None): + """Return list of names of the available models on this controller. :param bool all_: List all models, regardless of user accessibilty (admin use only) :param str username: User for which to list models (admin use only) """ - controller_facade = client.ControllerFacade.from_connection( - self.connection) - return await controller_facade.AllModels() + uuids = await self._model_uuids(all_, username) + return sorted(uuids.keys()) def get_payloads(self, *patterns): """Return list of known payloads. @@ -261,14 +328,6 @@ class Controller(object): """ raise NotImplementedError() - def get_users(self, all_=False): - """Return list of users that can connect to this controller. - - :param bool all_: Include disabled users - - """ - raise NotImplementedError() - def login(self): """Log in to this controller. @@ -284,25 +343,62 @@ class Controller(object): """ raise NotImplementedError() - def get_model(self, name): - """Get a model by name. + async def get_model(self, model): + """Get a model by name or UUID. - :param str name: Model name + :param str model: Model name or UUID """ - raise NotImplementedError() + uuids = await self._model_uuids() + if model in uuids: + name_or_uuid = uuids[model] + else: + name_or_uuid = model - async def get_user(self, username, include_disabled=False): + model = Model() + await model.connect( + self.connection.endpoint, + name_or_uuid, + self.connection.username, + self.connection.password, + self.connection.cacert, + self.connection.macaroons, + loop=self.loop, + ) + return model + + async def get_user(self, username): """Get a user by name. :param str username: Username - + :returns: A :class:`~juju.user.User` instance """ client_facade = client.UserManagerFacade.from_connection( self.connection) user = tag.user(username) - return await client_facade.UserInfo([client.Entity(user)], - include_disabled) + args = [client.Entity(user)] + try: + response = await client_facade.UserInfo(args, True) + except errors.JujuError as e: + if 'permission denied' in e.errors: + # apparently, trying to get info for a nonexistent user returns + # a "permission denied" error rather than an empty result set + return None + raise + if response.results and response.results[0].result: + return User(self, response.results[0].result) + return None + + async def get_users(self, include_disabled=False): + """Return list of users that can connect to this controller. + + :param bool include_disabled: Include disabled users + :returns: A list of :class:`~juju.user.User` instances + """ + client_facade = client.UserManagerFacade.from_connection( + self.connection) + response = await client_facade.UserInfo(None, include_disabled) + return [User(self, r.result) for r in response.results] async def grant(self, username, acl='login'): """Set access level of the given user on the controller diff --git a/modules/libjuju/juju/errors.py b/modules/libjuju/juju/errors.py index de52174..ecd1c0d 100644 --- a/modules/libjuju/juju/errors.py +++ b/modules/libjuju/juju/errors.py @@ -1,5 +1,16 @@ class JujuError(Exception): - pass + def __init__(self, *args, **kwargs): + self.message = '' + self.errors = [] + if args: + self.message = str(args[0]) + if isinstance(args[0], (list, tuple)): + self.errors = args[0] + elif len(args) > 1: + self.errors = list(args) + else: + self.errors = [self.message] + super().__init__(*args, **kwargs) class JujuAPIError(JujuError): diff --git a/modules/libjuju/juju/loop.py b/modules/libjuju/juju/loop.py index 4abedfc..aca726b 100644 --- a/modules/libjuju/juju/loop.py +++ b/modules/libjuju/juju/loop.py @@ -24,7 +24,7 @@ def run(*steps): try: loop.add_signal_handler(signal.SIGINT, abort) added = True - except ValueError as e: + except (ValueError, OSError, RuntimeError) as e: # add_signal_handler doesn't work in a thread if 'main thread' not in str(e): raise diff --git a/modules/libjuju/juju/model.py b/modules/libjuju/juju/model.py index bd8709a..fc8d5e9 100644 --- a/modules/libjuju/juju/model.py +++ b/modules/libjuju/juju/model.py @@ -1402,7 +1402,7 @@ class Model(object): key_facade = client.KeyManagerFacade.from_connection(self.connection) key = base64.b64decode(bytes(key.strip().split()[1].encode('ascii'))) key = hashlib.md5(key).hexdigest() - key = ':'.join(a+b for a, b in zip(key[::2], key[1::2])) + key = ':'.join(a + b for a, b in zip(key[::2], key[1::2])) await key_facade.DeleteKeys([key], user) remove_ssh_keys = remove_ssh_key @@ -1658,8 +1658,9 @@ class BundleHandler(object): apps, args = [], [] default_series = bundle.get('series') + apps_dict = bundle.get('applications', bundle.get('services', {})) for app_name in self.applications: - app_dict = bundle['services'][app_name] + app_dict = apps_dict[app_name] charm_dir = os.path.abspath(os.path.expanduser(app_dict['charm'])) if not os.path.isdir(charm_dir): continue @@ -1688,7 +1689,7 @@ class BundleHandler(object): ], loop=self.model.loop) # Update the 'charm:' entry for each app with the new 'local:' url. for app_name, charm_url in zip(apps, charm_urls): - bundle['services'][app_name]['charm'] = charm_url + apps_dict[app_name]['charm'] = charm_url return bundle @@ -1714,7 +1715,9 @@ class BundleHandler(object): @property def applications(self): - return list(self.bundle['services'].keys()) + apps_dict = self.bundle.get('applications', + self.bundle.get('services', {})) + return list(apps_dict.keys()) def resolve(self, reference): if reference and reference.startswith('$'): diff --git a/modules/libjuju/juju/utils.py b/modules/libjuju/juju/utils.py index 1d1b24e..1d9bc1c 100644 --- a/modules/libjuju/juju/utils.py +++ b/modules/libjuju/juju/utils.py @@ -11,11 +11,11 @@ async def execute_process(*cmd, log=None, loop=None): ''' p = await asyncio.create_subprocess_exec( - *cmd, - stdin=asyncio.subprocess.PIPE, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - loop=loop) + *cmd, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + loop=loop) stdout, stderr = await p.communicate() if log: log.debug("Exec %s -> %d", cmd, p.returncode) diff --git a/modules/libjuju/tests/base.py b/modules/libjuju/tests/base.py index e1ec452..96ed9c7 100644 --- a/modules/libjuju/tests/base.py +++ b/modules/libjuju/tests/base.py @@ -34,15 +34,22 @@ class CleanController(): class CleanModel(): def __init__(self): + self.user_name = None self.controller = None + self.controller_name = None self.model = None + self.model_name = None + self.model_uuid = None async def __aenter__(self): self.controller = Controller() - await self.controller.connect_current() + juju_data = JujuData() + self.controller_name = juju_data.current_controller() + self.user_name = juju_data.accounts()[self.controller_name]['user'] + await self.controller.connect_controller(self.controller_name) - model_name = 'model-{}'.format(uuid.uuid4()) - self.model = await self.controller.add_model(model_name) + self.model_name = 'test-{}'.format(uuid.uuid4()) + self.model = await self.controller.add_model(self.model_name) # save the model UUID in case test closes model self.model_uuid = self.model.info.uuid @@ -50,12 +57,28 @@ class CleanModel(): # Ensure that we connect to the new model by default. This also # prevents failures if test was started with no current model. self._patch_cm = mock.patch.object(JujuData, 'current_model', - return_value=model_name) + return_value=self.model_name) self._patch_cm.start() + # Ensure that the models data includes this model, since it doesn't + # get added to the client store by Controller.add_model(). + self._orig_models = JujuData().models + self._patch_models = mock.patch.object(JujuData, 'models', + side_effect=self._models) + self._patch_models.start() + return self.model + def _models(self): + result = self._orig_models() + models = result[self.controller_name]['models'] + full_model_name = '{}/{}'.format(self.user_name, self.model_name) + if full_model_name not in models: + models[full_model_name] = {'uuid': self.model_uuid} + return result + async def __aexit__(self, exc_type, exc, tb): + self._patch_models.stop() self._patch_cm.stop() await self.model.disconnect() await self.controller.destroy_model(self.model_uuid) diff --git a/modules/libjuju/tests/integration/test_application.py b/modules/libjuju/tests/integration/test_application.py index 1a4fcaa..7b780da 100644 --- a/modules/libjuju/tests/integration/test_application.py +++ b/modules/libjuju/tests/integration/test_application.py @@ -1,3 +1,4 @@ +import asyncio import pytest from .. import base @@ -92,3 +93,32 @@ async def test_upgrade_charm_switch(event_loop): assert app.data['charm-url'] == 'cs:ubuntu-0' await app.upgrade_charm(switch='ubuntu-8') assert app.data['charm-url'] == 'cs:ubuntu-8' + + +@base.bootstrapped +@pytest.mark.asyncio +async def test_upgrade_charm_resource(event_loop): + async with base.CleanModel() as model: + app = await model.deploy('cs:~cynerva/upgrade-charm-resource-test-1') + + def units_ready(): + if not app.units: + return False + unit = app.units[0] + return unit.workload_status == 'active' and \ + unit.agent_status == 'idle' + + await asyncio.wait_for(model.block_until(units_ready), timeout=480) + unit = app.units[0] + expected_message = 'I have no resource.' + assert unit.workload_status_message == expected_message + + await app.upgrade_charm(revision=2) + await asyncio.wait_for( + model.block_until( + lambda: unit.workload_status_message != 'I have no resource.' + ), + timeout=60 + ) + expected_message = 'My resource: I am the resource.' + assert app.units[0].workload_status_message == expected_message diff --git a/modules/libjuju/tests/integration/test_controller.py b/modules/libjuju/tests/integration/test_controller.py index f3840cc..d559313 100644 --- a/modules/libjuju/tests/integration/test_controller.py +++ b/modules/libjuju/tests/integration/test_controller.py @@ -1,3 +1,4 @@ +import asyncio import pytest import uuid @@ -8,13 +9,21 @@ from juju.errors import JujuAPIError @base.bootstrapped @pytest.mark.asyncio -async def test_add_user(event_loop): +async def test_add_remove_user(event_loop): async with base.CleanController() as controller: username = 'test{}'.format(uuid.uuid4()) - await controller.add_user(username) - result = await controller.get_user(username) - res_ser = result.serialize()['results'][0].serialize() - assert res_ser['result'] is not None + user = await controller.get_user(username) + assert user is None + user = await controller.add_user(username) + assert user is not None + assert user.username == username + users = await controller.get_users() + assert any(u.username == username for u in users) + await controller.remove_user(username) + user = await controller.get_user(username) + assert user is None + users = await controller.get_users() + assert not any(u.username == username for u in users) @base.bootstrapped @@ -22,15 +31,23 @@ async def test_add_user(event_loop): async def test_disable_enable_user(event_loop): async with base.CleanController() as controller: username = 'test-disable{}'.format(uuid.uuid4()) - await controller.add_user(username) - await controller.disable_user(username) - result = await controller.get_user(username) - res_ser = result.serialize()['results'][0].serialize() - assert res_ser['result'].serialize()['disabled'] is True - await controller.enable_user(username) - result = await controller.get_user(username) - res_ser = result.serialize()['results'][0].serialize() - assert res_ser['result'].serialize()['disabled'] is False + user = await controller.add_user(username) + + await user.disable() + assert not user.enabled + assert user.disabled + + fresh = await controller.get_user(username) # fetch fresh copy + assert not fresh.enabled + assert fresh.disabled + + await user.enable() + assert user.enabled + assert not user.disabled + + fresh = await controller.get_user(username) # fetch fresh copy + assert fresh.enabled + assert not fresh.disabled @base.bootstrapped @@ -38,40 +55,98 @@ async def test_disable_enable_user(event_loop): async def test_change_user_password(event_loop): async with base.CleanController() as controller: username = 'test-password{}'.format(uuid.uuid4()) - await controller.add_user(username) - await controller.change_user_password(username, 'password') + user = await controller.add_user(username) + await user.set_password('password') try: new_controller = Controller() await new_controller.connect( controller.connection.endpoint, username, 'password') - result = True - await new_controller.disconnect() except JujuAPIError: - result = False - assert result is True + raise AssertionError('Unable to connect with new password') + finally: + await new_controller.disconnect() @base.bootstrapped @pytest.mark.asyncio -async def test_grant(event_loop): +async def test_grant_revoke(event_loop): async with base.CleanController() as controller: username = 'test-grant{}'.format(uuid.uuid4()) - await controller.add_user(username) - await controller.grant(username, 'superuser') - result = await controller.get_user(username) - result = result.serialize()['results'][0].serialize()['result']\ - .serialize() - assert result['access'] == 'superuser' - await controller.grant(username, 'login') - result = await controller.get_user(username) - result = result.serialize()['results'][0].serialize()['result']\ - .serialize() - assert result['access'] == 'login' + user = await controller.add_user(username) + await user.grant('superuser') + assert user.access == 'superuser' + fresh = await controller.get_user(username) # fetch fresh copy + assert fresh.access == 'superuser' + await user.grant('login') + assert user.access == 'login' + fresh = await controller.get_user(username) # fetch fresh copy + assert fresh.access == 'login' + await user.revoke() + assert user.access is '' + fresh = await controller.get_user(username) # fetch fresh copy + assert fresh.access is '' + + +@base.bootstrapped +@pytest.mark.asyncio +async def test_list_models(event_loop): + async with base.CleanController() as controller: + async with base.CleanModel() as model: + result = await controller.list_models() + assert model.info.name in result + + +@base.bootstrapped +@pytest.mark.asyncio +async def test_get_model(event_loop): + async with base.CleanController() as controller: + by_name, by_uuid = None, None + model_name = 'test-{}'.format(uuid.uuid4()) + model = await controller.add_model(model_name) + model_uuid = model.info.uuid + await model.disconnect() + try: + by_name = await controller.get_model(model_name) + by_uuid = await controller.get_model(model_uuid) + assert by_name.info.name == model_name + assert by_name.info.uuid == model_uuid + assert by_uuid.info.name == model_name + assert by_uuid.info.uuid == model_uuid + finally: + if by_name: + await by_name.disconnect() + if by_uuid: + await by_uuid.disconnect() + await controller.destroy_model(model_name) + + +async def _wait_for_model_gone(controller, model_name): + while model_name in await controller.list_models(): + await asyncio.sleep(0.5, loop=controller.loop) + + +@base.bootstrapped +@pytest.mark.asyncio +async def test_destroy_model_by_name(event_loop): + async with base.CleanController() as controller: + model_name = 'test-{}'.format(uuid.uuid4()) + model = await controller.add_model(model_name) + await model.disconnect() + await controller.destroy_model(model_name) + await asyncio.wait_for(_wait_for_model_gone(controller, + model_name), + timeout=60) @base.bootstrapped @pytest.mark.asyncio -async def test_get_models(event_loop): +async def test_add_destroy_model_by_uuid(event_loop): async with base.CleanController() as controller: - result = await controller.get_models() - assert isinstance(result.serialize()['user-models'], list) + model_name = 'test-{}'.format(uuid.uuid4()) + model = await controller.add_model(model_name) + model_uuid = model.info.uuid + await model.disconnect() + await controller.destroy_model(model_uuid) + await asyncio.wait_for(_wait_for_model_gone(controller, + model_name), + timeout=60) diff --git a/modules/libjuju/tests/integration/test_machine.py b/modules/libjuju/tests/integration/test_machine.py index 60de035..cabf46d 100644 --- a/modules/libjuju/tests/integration/test_machine.py +++ b/modules/libjuju/tests/integration/test_machine.py @@ -28,7 +28,8 @@ async def test_status(event_loop): await asyncio.wait_for( model.block_until(lambda: (machine.status == 'running' and - machine.agent_status == 'started')), + machine.agent_status == 'started' and + machine.agent_version is not None)), timeout=480) assert machine.status == 'running' diff --git a/modules/libjuju/tests/integration/test_model.py b/modules/libjuju/tests/integration/test_model.py index 8506786..041f75a 100644 --- a/modules/libjuju/tests/integration/test_model.py +++ b/modules/libjuju/tests/integration/test_model.py @@ -129,18 +129,6 @@ async def _deploy_in_loop(new_loop, model_name): await new_model.disconnect() -@base.bootstrapped -@pytest.mark.asyncio -async def test_explicit_loop(event_loop): - async with base.CleanModel() as model: - model_name = model.info.name - new_loop = asyncio.new_event_loop() - new_loop.run_until_complete( - _deploy_in_loop(new_loop, model_name)) - await model._wait_for_new('application', 'ubuntu') - assert 'ubuntu' in model.applications - - @base.bootstrapped @pytest.mark.asyncio async def test_explicit_loop_threaded(event_loop): diff --git a/modules/libjuju/tests/unit/test_client.py b/modules/libjuju/tests/unit/test_client.py index 7828cf3..e9fde8e 100644 --- a/modules/libjuju/tests/unit/test_client.py +++ b/modules/libjuju/tests/unit/test_client.py @@ -4,16 +4,14 @@ Tests for generated client code """ import mock -import pytest from juju.client import client - def test_basics(): assert client.CLIENTS - for i in range(1,5): # Assert versions 1-4 in client dict + for i in range(1, 5): # Assert versions 1-4 in client dict assert str(i) in client.CLIENTS @@ -21,5 +19,10 @@ def test_from_connection(): connection = mock.Mock() connection.facades = {"Action": 2} action_facade = client.ActionFacade.from_connection(connection) + assert action_facade + - +def test_to_json(): + uml = client.UserModelList([client.UserModel()]) + assert uml.to_json() == ('{"user-models": [{"last-connection": null, ' + '"model": null}]}') diff --git a/modules/libjuju/tox.ini b/modules/libjuju/tox.ini index 789bbeb..ce421d6 100644 --- a/modules/libjuju/tox.ini +++ b/modules/libjuju/tox.ini @@ -4,11 +4,14 @@ # and then run "tox" from this directory. [tox] -envlist = py35 +envlist = lint,py35 skipsdist=True [testenv] +basepython=python3 usedevelop=True +# for testing with other python versions +commands = py.test -ra -v -s -x -n auto {posargs} passenv = HOME deps = @@ -23,6 +26,15 @@ deps = # default tox env excludes integration tests commands = py.test -ra -v -s -x -n auto -k 'not integration' {posargs} +[testenv:lint] +envdir = {toxworkdir}/py35 +commands = + flake8 --ignore E501 {posargs} juju +deps = + flake8 + [testenv:integration] -basepython=python3 -commands = py.test -ra -v -s -x -n auto {posargs} +envdir = {toxworkdir}/py35 + +[flake8] +exclude = juju/client/_* -- 2.17.1