Improved Primitive support and better testing
This changeset addresses several issues.
- Improve primitive support so the status and output of an executed
primitive can be retrieved
- Merge latest upstream libjuju (required for new primive features)
- New testing framework
This is the start of a new testing framework with the ability to
create and configure LXD containers with SSH, to use while testing proxy
charms.
- Add support for using ssh keys with proxy charms
See Feature 1429. This uses the per-proxy charm/unit ssh keypair
Signed-off-by: Adam Israel <adam.israel@canonical.com>
diff --git a/modules/libjuju/juju/application.py b/modules/libjuju/juju/application.py
index 555bb3d..84afebe 100644
--- a/modules/libjuju/juju/application.py
+++ b/modules/libjuju/juju/application.py
@@ -228,13 +228,24 @@
result = (await app_facade.Get(self.name)).constraints
return vars(result) if result else result
- def get_actions(self, schema=False):
+ async def get_actions(self, schema=False):
"""Get actions defined for this application.
:param bool schema: Return the full action schema
-
+ :return dict: The charms actions, empty dict if none are defined.
"""
- raise NotImplementedError()
+ actions = {}
+ entity = [{"tag": self.tag}]
+ action_facade = client.ActionFacade.from_connection(self.connection)
+ results = (
+ await action_facade.ApplicationsCharmsActions(entity)).results
+ for result in results:
+ if result.application_tag == self.tag and result.actions:
+ actions = result.actions
+ break
+ if not schema:
+ actions = {k: v['description'] for k, v in actions.items()}
+ return actions
def get_resources(self, details=False):
"""Return resources for this application.
@@ -284,12 +295,10 @@
)
return await self.ann_facade.Set([ann])
- async def set_config(self, config, to_default=False):
+ async def set_config(self, config):
"""Set configuration options for this application.
:param config: Dict of configuration to set
- :param bool to_default: Set application options to default values
-
"""
app_facade = client.ApplicationFacade.from_connection(self.connection)
@@ -298,6 +307,19 @@
return await app_facade.Set(self.name, config)
+ async def reset_config(self, to_default):
+ """
+ Restore application config to default values.
+
+ :param list to_default: A list of config options to be reset to their default value.
+ """
+ app_facade = client.ApplicationFacade.from_connection(self.connection)
+
+ log.debug(
+ 'Restoring default config for %s: %s', self.name, to_default)
+
+ return await app_facade.Unset(self.name, to_default)
+
async def set_constraints(self, constraints):
"""Set machine constraints for this application.
diff --git a/modules/libjuju/juju/client/connection.py b/modules/libjuju/juju/client/connection.py
index bdd1c3f..13770a5 100644
--- a/modules/libjuju/juju/client/connection.py
+++ b/modules/libjuju/juju/client/connection.py
@@ -109,20 +109,22 @@
If uuid is None, the connection will be to the controller. Otherwise it
will be to the model.
- :param str endpoint The hostname:port of the controller to connect to.
- :param str uuid The model UUID to connect to (None for a
+
+ :param str endpoint: The hostname:port of the controller to connect to.
+ :param str uuid: The model UUID to connect to (None for a
controller-only connection).
- :param str username The username for controller-local users (or None
+ :param str username: The username for controller-local users (or None
to use macaroon-based login.)
- :param str password The password for controller-local users.
- :param str cacert The CA certificate of the controller (PEM formatted).
- :param httpbakery.Client bakery_client The macaroon bakery client to
+ :param str password: The password for controller-local users.
+ :param str cacert: The CA certificate of the controller
+ (PEM formatted).
+ :param httpbakery.Client bakery_client: The macaroon bakery client to
to use when performing macaroon-based login. Macaroon tokens
acquired when logging will be saved to bakery_client.cookies.
If this is None, a default bakery_client will be used.
- :param loop asyncio.BaseEventLoop The event loop to use for async
+ :param asyncio.BaseEventLoop loop: The event loop to use for async
operations.
- :param max_frame_size The maximum websocket frame size to allow.
+ :param int max_frame_size: The maximum websocket frame size to allow.
"""
self = cls()
if endpoint is None:
@@ -519,8 +521,8 @@
async def login(self):
params = {}
+ params['auth-tag'] = self.usertag
if self.password:
- params['auth-tag'] = self.usertag
params['credentials'] = self.password
else:
macaroons = _macaroons_for_domain(self.bakery_client.cookies,
diff --git a/modules/libjuju/juju/client/connector.py b/modules/libjuju/juju/client/connector.py
index 64fbe44..a30adbf 100644
--- a/modules/libjuju/juju/client/connector.py
+++ b/modules/libjuju/juju/client/connector.py
@@ -4,6 +4,7 @@
import macaroonbakery.httpbakery as httpbakery
from juju.client.connection import Connection
+from juju.client.gocookies import go_to_py_cookie, GoCookieJar
from juju.client.jujudata import FileJujuData
from juju.errors import JujuConnectionError, JujuError
@@ -56,6 +57,14 @@
kwargs.setdefault('loop', self.loop)
kwargs.setdefault('max_frame_size', self.max_frame_size)
kwargs.setdefault('bakery_client', self.bakery_client)
+ if 'macaroons' in kwargs:
+ if not kwargs['bakery_client']:
+ kwargs['bakery_client'] = httpbakery.Client()
+ if not kwargs['bakery_client'].cookies:
+ kwargs['bakery_client'].cookies = GoCookieJar()
+ jar = kwargs['bakery_client'].cookies
+ for macaroon in kwargs.pop('macaroons'):
+ jar.set_cookie(go_to_py_cookie(macaroon))
self._connection = await Connection.connect(**kwargs)
async def disconnect(self):
diff --git a/modules/libjuju/juju/client/facade.py b/modules/libjuju/juju/client/facade.py
index 1c7baa0..9e2aabf 100644
--- a/modules/libjuju/juju/client/facade.py
+++ b/modules/libjuju/juju/client/facade.py
@@ -171,13 +171,13 @@
def strcast(kind, keep_builtins=False):
- if issubclass(kind, typing.GenericMeta):
- return str(kind)[1:]
- if str(kind).startswith('~'):
- return str(kind)[1:]
if (kind in basic_types or
type(kind) in basic_types) and keep_builtins is False:
return kind.__name__
+ if str(kind).startswith('~'):
+ return str(kind)[1:]
+ if issubclass(kind, typing.GenericMeta):
+ return str(kind)[1:]
return kind
@@ -291,6 +291,13 @@
source.append("{}self.{} = {}".format(INDENT * 2,
arg_name,
arg_name))
+ elif type(arg_type) is typing.TypeVar:
+ source.append("{}self.{} = {}.from_json({}) "
+ "if {} else None".format(INDENT * 2,
+ arg_name,
+ arg_type_name,
+ arg_name,
+ arg_name))
elif issubclass(arg_type, typing.Sequence):
value_type = (
arg_type_name.__parameters__[0]
@@ -326,13 +333,6 @@
source.append("{}self.{} = {}".format(INDENT * 2,
arg_name,
arg_name))
- elif type(arg_type) is typing.TypeVar:
- source.append("{}self.{} = {}.from_json({}) "
- "if {} else None".format(INDENT * 2,
- arg_name,
- arg_type_name,
- arg_name,
- arg_name))
else:
source.append("{}self.{} = {}".format(INDENT * 2,
arg_name,
@@ -434,7 +434,7 @@
return decorator
-def makeFunc(cls, name, params, result, async=True):
+def makeFunc(cls, name, params, result, _async=True):
INDENT = " "
args = Args(params)
assignments = []
@@ -448,7 +448,7 @@
source = """
@ReturnMapping({rettype})
-{async}def {name}(self{argsep}{args}):
+{_async}def {name}(self{argsep}{args}):
'''
{docstring}
Returns -> {res}
@@ -460,12 +460,12 @@
version={cls.version},
params=_params)
{assignments}
- reply = {await}self.rpc(msg)
+ reply = {_await}self.rpc(msg)
return reply
"""
- fsource = source.format(async="async " if async else "",
+ fsource = source.format(_async="async " if _async else "",
name=name,
argsep=", " if args else "",
args=args,
@@ -474,7 +474,7 @@
docstring=textwrap.indent(args.get_doc(), INDENT),
cls=cls,
assignments=assignments,
- await="await " if async else "")
+ _await="await " if _async else "")
ns = _getns()
exec(fsource, ns)
func = ns[name]
diff --git a/modules/libjuju/juju/client/gocookies.py b/modules/libjuju/juju/client/gocookies.py
index a8a0df8..3e48b8d 100644
--- a/modules/libjuju/juju/client/gocookies.py
+++ b/modules/libjuju/juju/client/gocookies.py
@@ -15,7 +15,7 @@
to implement the actual cookie loading'''
data = json.load(f) or []
now = time.time()
- for cookie in map(_new_py_cookie, data):
+ for cookie in map(go_to_py_cookie, data):
if not ignore_expires and cookie.is_expired(now):
continue
self.set_cookie(cookie)
@@ -37,12 +37,12 @@
continue
if not ignore_expires and cookie.is_expired(now):
continue
- go_cookies.append(_new_go_cookie(cookie))
+ go_cookies.append(py_to_go_cookie(cookie))
with open(filename, "w") as f:
f.write(json.dumps(go_cookies))
-def _new_py_cookie(go_cookie):
+def go_to_py_cookie(go_cookie):
'''Convert a Go-style JSON-unmarshaled cookie into a Python cookie'''
expires = None
if go_cookie.get('Expires') is not None:
@@ -75,7 +75,7 @@
)
-def _new_go_cookie(py_cookie):
+def py_to_go_cookie(py_cookie):
'''Convert a python cookie to the JSON-marshalable Go-style cookie form.'''
# TODO (perhaps):
# HttpOnly
diff --git a/modules/libjuju/juju/client/overrides.py b/modules/libjuju/juju/client/overrides.py
index 8b29de7..49ab931 100644
--- a/modules/libjuju/juju/client/overrides.py
+++ b/modules/libjuju/juju/client/overrides.py
@@ -15,6 +15,7 @@
__patches__ = [
'ResourcesFacade',
'AllWatcherFacade',
+ 'ActionFacade',
]
@@ -105,6 +106,42 @@
return result
+class ActionFacade(Type):
+
+ class _FindTagsResults(Type):
+ _toSchema = {'matches': 'matches'}
+ _toPy = {'matches': 'matches'}
+
+ def __init__(self, matches=None, **unknown_fields):
+ '''
+ FindTagsResults wraps the mapping between the requested prefix and the
+ matching tags for each requested prefix.
+
+ Matches map[string][]Entity `json:"matches"`
+ '''
+ self.matches = {}
+ matches = matches or {}
+ for prefix, tags in matches.items():
+ self.matches[prefix] = [_definitions.Entity.from_json(r)
+ for r in tags]
+
+ @ReturnMapping(_FindTagsResults)
+ async def FindActionTagsByPrefix(self, prefixes):
+ '''
+ prefixes : typing.Sequence[str]
+ Returns -> typing.Sequence[~Entity]
+ '''
+ # map input types to rpc msg
+ _params = dict()
+ msg = dict(type='Action',
+ request='FindActionTagsByPrefix',
+ version=2,
+ params=_params)
+ _params['prefixes'] = prefixes
+ reply = await self.rpc(msg)
+ return reply
+
+
class Number(_definitions.Number):
"""
This type represents a semver string.
@@ -138,14 +175,24 @@
def __str__(self):
return self.serialize()
+ @property
+ def _cmp(self):
+ return (self.major, self.minor, self.tag, self.patch, self.build)
+
def __eq__(self, other):
- return (
- isinstance(other, type(self)) and
- other.major == self.major and
- other.minor == self.minor and
- other.tag == self.tag and
- other.patch == self.patch and
- other.build == self.build)
+ return isinstance(other, type(self)) and self._cmp == other._cmp
+
+ def __lt__(self, other):
+ return self._cmp < other._cmp
+
+ def __le__(self, other):
+ return self._cmp <= other._cmp
+
+ def __gt__(self, other):
+ return self._cmp > other._cmp
+
+ def __ge__(self, other):
+ return self._cmp >= other._cmp
@classmethod
def from_json(cls, data):
diff --git a/modules/libjuju/juju/constraints.py b/modules/libjuju/juju/constraints.py
index 998862d..0050673 100644
--- a/modules/libjuju/juju/constraints.py
+++ b/modules/libjuju/juju/constraints.py
@@ -29,6 +29,8 @@
"P": 1024 * 1024 * 1024
}
+LIST_KEYS = {'tags', 'spaces'}
+
SNAKE1 = re.compile(r'(.)([A-Z][a-z]+)')
SNAKE2 = re.compile('([a-z0-9])([A-Z])')
@@ -47,8 +49,10 @@
return constraints
constraints = {
- normalize_key(k): normalize_value(v) for k, v in [
- s.split("=") for s in constraints.split(" ")]}
+ normalize_key(k): (
+ normalize_list_value(v) if k in LIST_KEYS else
+ normalize_value(v)
+ ) for k, v in [s.split("=") for s in constraints.split(" ")]}
return constraints
@@ -72,13 +76,12 @@
# Translate aliases to Megabytes. e.g. 1G = 10240
return int(value[:-1]) * FACTORS[value[-1:]]
- if "," in value:
- # Handle csv strings.
- values = value.split(",")
- values = [normalize_value(v) for v in values]
- return values
-
if value.isdigit():
return int(value)
return value
+
+
+def normalize_list_value(value):
+ values = value.strip().split(',')
+ return [normalize_value(value) for value in values]
diff --git a/modules/libjuju/juju/controller.py b/modules/libjuju/juju/controller.py
index 957ab85..b4c544e 100644
--- a/modules/libjuju/juju/controller.py
+++ b/modules/libjuju/juju/controller.py
@@ -1,5 +1,7 @@
import asyncio
+import json
import logging
+from pathlib import Path
from . import errors, tag, utils
from .client import client, connector
@@ -48,32 +50,101 @@
def loop(self):
return self._connector.loop
- async def connect(self, controller_name=None, **kwargs):
+ async def connect(self, *args, **kwargs):
"""Connect to a Juju controller.
- If any arguments are specified other than controller_name,
- then controller_name must be None and an explicit
- connection will be made using Connection.connect
- using those parameters (the 'uuid' parameter must
- be absent or None).
+ This supports two calling conventions:
- Otherwise, if controller_name is None, connect to the
- current controller.
+ The controller and (optionally) authentication information can be
+ taken from the data files created by the Juju CLI. This convention
+ will be used if a ``controller_name`` is specified, or if the
+ ``endpoint`` is not.
- Otherwise, controller_name must specify the name
- of a known controller.
+ Otherwise, both the ``endpoint`` and authentication information
+ (``username`` and ``password``, or ``bakery_client`` and/or
+ ``macaroons``) are required.
+
+ If a single positional argument is given, it will be assumed to be
+ the ``controller_name``. Otherwise, the first positional argument,
+ if any, must be the ``endpoint``.
+
+ Available parameters are:
+
+ :param str controller_name: Name of controller registered with the
+ Juju CLI.
+ :param str endpoint: The hostname:port of the controller to connect to.
+ :param str username: The username for controller-local users (or None
+ to use macaroon-based login.)
+ :param str password: The password for controller-local users.
+ :param str cacert: The CA certificate of the controller
+ (PEM formatted).
+ :param httpbakery.Client bakery_client: The macaroon bakery client to
+ to use when performing macaroon-based login. Macaroon tokens
+ acquired when logging will be saved to bakery_client.cookies.
+ If this is None, a default bakery_client will be used.
+ :param list macaroons: List of macaroons to load into the
+ ``bakery_client``.
+ :param asyncio.BaseEventLoop loop: The event loop to use for async
+ operations.
+ :param int max_frame_size: The maximum websocket frame size to allow.
"""
await self.disconnect()
- if not kwargs:
- await self._connector.connect_controller(controller_name)
+ if 'endpoint' not in kwargs and len(args) < 2:
+ if args and 'model_name' in kwargs:
+ raise TypeError('connect() got multiple values for '
+ 'controller_name')
+ elif args:
+ controller_name = args[0]
+ else:
+ controller_name = kwargs.pop('controller_name', None)
+ await self._connector.connect_controller(controller_name, **kwargs)
else:
- if controller_name is not None:
- raise ValueError('controller name may not be specified with other connect parameters')
- if kwargs.get('uuid') is not None:
- # A UUID implies a model connection, not a controller connection.
- raise ValueError('model UUID specified when connecting to controller')
+ if 'controller_name' in kwargs:
+ raise TypeError('connect() got values for both '
+ 'controller_name and endpoint')
+ if args and 'endpoint' in kwargs:
+ raise TypeError('connect() got multiple values for endpoint')
+ has_userpass = (len(args) >= 3 or
+ {'username', 'password'}.issubset(kwargs))
+ has_macaroons = (len(args) >= 5 or not
+ {'bakery_client', 'macaroons'}.isdisjoint(kwargs))
+ if not (has_userpass or has_macaroons):
+ raise TypeError('connect() missing auth params')
+ arg_names = [
+ 'endpoint',
+ 'username',
+ 'password',
+ 'cacert',
+ 'bakery_client',
+ 'macaroons',
+ 'loop',
+ 'max_frame_size',
+ ]
+ for i, arg in enumerate(args):
+ kwargs[arg_names[i]] = arg
+ if 'endpoint' not in kwargs:
+ raise ValueError('endpoint is required '
+ 'if controller_name not given')
+ if not ({'username', 'password'}.issubset(kwargs) or
+ {'bakery_client', 'macaroons'}.intersection(kwargs)):
+ raise ValueError('Authentication parameters are required '
+ 'if controller_name not given')
await self._connector.connect(**kwargs)
+ async def connect_current(self):
+ """
+ .. deprecated:: 0.7.3
+ Use :meth:`.connect()` instead.
+ """
+ return await self.connect()
+
+ async def connect_controller(self, controller_name):
+ """
+ .. deprecated:: 0.7.3
+ Use :meth:`.connect(controller_name)` instead.
+ """
+ return await self.connect(controller_name)
+
async def _connect_direct(self, **kwargs):
await self.disconnect()
await self._connector.connect(**kwargs)
@@ -127,6 +198,21 @@
raise errors.JujuError(
'Unable to find credential: {}'.format(name))
+ if credential.auth_type == 'jsonfile' and 'file' in credential.attrs:
+ # file creds have to be loaded before being sent to the controller
+ try:
+ # it might already be JSON
+ json.loads(credential.attrs['file'])
+ except json.JSONDecodeError:
+ # not valid JSON, so maybe it's a file
+ cred_path = Path(credential.attrs['file'])
+ if cred_path.exists():
+ # make a copy
+ cred_json = credential.to_json()
+ credential = client.CloudCredential.from_json(cred_json)
+ # inline the cred
+ credential.attrs['file'] = cred_path.read_text()
+
log.debug('Uploading credential %s', name)
cloud_facade = client.CloudFacade.from_connection(self.connection())
await cloud_facade.UpdateCredentials([
@@ -315,7 +401,7 @@
Use :meth:`.list_models` instead.
"""
controller_facade = client.ControllerFacade.from_connection(
- self.connection)
+ self.connection())
for attempt in (1, 2, 3):
try:
return await controller_facade.AllModels()
diff --git a/modules/libjuju/juju/machine.py b/modules/libjuju/juju/machine.py
index bd3d030..a46135c 100644
--- a/modules/libjuju/juju/machine.py
+++ b/modules/libjuju/juju/machine.py
@@ -14,7 +14,18 @@
class Machine(model.ModelEntity):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
- self.on_change(self._workaround_1695335)
+ self.model.loop.create_task(self._queue_workarounds())
+
+ async def _queue_workarounds(self):
+ model = self.model
+ if not model.info:
+ await utils.run_with_interrupt(model.get_info(),
+ model._watch_stopping,
+ model.loop)
+ if model._watch_stopping.is_set():
+ return
+ if model.info.agent_version < client.Number.from_json('2.2.3'):
+ self.on_change(self._workaround_1695335)
async def _workaround_1695335(self, delta, old, new, model):
"""
diff --git a/modules/libjuju/juju/model.py b/modules/libjuju/juju/model.py
index ac22599..37e8cd6 100644
--- a/modules/libjuju/juju/model.py
+++ b/modules/libjuju/juju/model.py
@@ -22,12 +22,15 @@
from . import tag, utils
from .client import client, connector
from .client.client import ConfigValue
+from .client.client import Value
from .constraints import parse as parse_constraints
from .constraints import normalize_key
from .delta import get_entity_class, get_entity_delta
from .errors import JujuAPIError, JujuError
from .exceptions import DeadEntityException
from .placement import parse as parse_placement
+from . import provisioner
+
log = logging.getLogger(__name__)
@@ -410,7 +413,7 @@
`juju.client.connection.Connection.MAX_FRAME_SIZE`
:param bakery_client httpbakery.Client: The bakery client to use
for macaroon authorization.
- :param jujudata JujuData: The source for current controller information.
+ :param jujudata JujuData: The source for current controller information
"""
self._connector = connector.Connector(
loop=loop,
@@ -458,42 +461,101 @@
async def __aexit__(self, exc_type, exc, tb):
await self.disconnect()
- async def connect(self, model_name=None, **kwargs):
+ async def connect(self, *args, **kwargs):
"""Connect to a juju model.
- If any arguments are specified other than model_name, then
- model_name must be None and an explicit connection will be made
- using Connection.connect using those parameters (the 'uuid'
- parameter must be specified).
+ This supports two calling conventions:
- Otherwise, if model_name is None, connect to the current model.
+ The model and (optionally) authentication information can be taken
+ from the data files created by the Juju CLI. This convention will
+ be used if a ``model_name`` is specified, or if the ``endpoint``
+ and ``uuid`` are not.
- Otherwise, model_name must specify the name of a known
- model.
+ Otherwise, all of the ``endpoint``, ``uuid``, and authentication
+ information (``username`` and ``password``, or ``bakery_client`` and/or
+ ``macaroons``) are required.
+
+ If a single positional argument is given, it will be assumed to be
+ the ``model_name``. Otherwise, the first positional argument, if any,
+ must be the ``endpoint``.
+
+ Available parameters are:
:param model_name: Format [controller:][user/]model
-
+ :param str endpoint: The hostname:port of the controller to connect to.
+ :param str uuid: The model UUID to connect to.
+ :param str username: The username for controller-local users (or None
+ to use macaroon-based login.)
+ :param str password: The password for controller-local users.
+ :param str cacert: The CA certificate of the controller
+ (PEM formatted).
+ :param httpbakery.Client bakery_client: The macaroon bakery client to
+ to use when performing macaroon-based login. Macaroon tokens
+ acquired when logging will be saved to bakery_client.cookies.
+ If this is None, a default bakery_client will be used.
+ :param list macaroons: List of macaroons to load into the
+ ``bakery_client``.
+ :param asyncio.BaseEventLoop loop: The event loop to use for async
+ operations.
+ :param int max_frame_size: The maximum websocket frame size to allow.
"""
await self.disconnect()
- if not kwargs:
- await self._connector.connect_model(model_name)
+ if 'endpoint' not in kwargs and len(args) < 2:
+ if args and 'model_name' in kwargs:
+ raise TypeError('connect() got multiple values for model_name')
+ elif args:
+ model_name = args[0]
+ else:
+ model_name = kwargs.pop('model_name', None)
+ await self._connector.connect_model(model_name, **kwargs)
else:
- if kwargs.get('uuid') is None:
- raise ValueError('no UUID specified when connecting to model')
+ if 'model_name' in kwargs:
+ raise TypeError('connect() got values for both '
+ 'model_name and endpoint')
+ if args and 'endpoint' in kwargs:
+ raise TypeError('connect() got multiple values for endpoint')
+ if len(args) < 2 and 'uuid' not in kwargs:
+ raise TypeError('connect() missing value for uuid')
+ has_userpass = (len(args) >= 4 or
+ {'username', 'password'}.issubset(kwargs))
+ has_macaroons = (len(args) >= 6 or not
+ {'bakery_client', 'macaroons'}.isdisjoint(kwargs))
+ if not (has_userpass or has_macaroons):
+ raise TypeError('connect() missing auth params')
+ arg_names = [
+ 'endpoint',
+ 'uuid',
+ 'username',
+ 'password',
+ 'cacert',
+ 'bakery_client',
+ 'macaroons',
+ 'loop',
+ 'max_frame_size',
+ ]
+ for i, arg in enumerate(args):
+ kwargs[arg_names[i]] = arg
+ if not {'endpoint', 'uuid'}.issubset(kwargs):
+ raise ValueError('endpoint and uuid are required '
+ 'if model_name not given')
+ if not ({'username', 'password'}.issubset(kwargs) or
+ {'bakery_client', 'macaroons'}.intersection(kwargs)):
+ raise ValueError('Authentication parameters are required '
+ 'if model_name not given')
await self._connector.connect(**kwargs)
await self._after_connect()
async def connect_model(self, model_name):
"""
.. deprecated:: 0.6.2
- Use connect(model_name=model_name) instead.
+ Use ``connect(model_name=model_name)`` instead.
"""
return await self.connect(model_name=model_name)
async def connect_current(self):
"""
.. deprecated:: 0.6.2
- Use connect instead.
+ Use ``connect()`` instead.
"""
return await self.connect()
@@ -528,7 +590,7 @@
if self.is_connected():
log.debug('Closing model connection')
await self._connector.disconnect()
- self.info = None
+ self._info = None
async def add_local_charm_dir(self, charm_dir, series):
"""Upload a local charm to the model.
@@ -675,11 +737,19 @@
"""
facade = client.ClientFacade.from_connection(self.connection())
- self.info = await facade.ModelInfo()
+ self._info = await facade.ModelInfo()
log.debug('Got ModelInfo: %s', vars(self.info))
return self.info
+ @property
+ def info(self):
+ """Return the cached client.ModelInfo object for this Model.
+
+ If Model.get_info() has not been called, this will return None.
+ """
+ return self._info
+
def add_observer(
self, callable_, entity_type=None, action=None, entity_id=None,
predicate=None):
@@ -880,7 +950,8 @@
(None) - starts a new machine
'lxd' - starts a new machine with one lxd container
'lxd:4' - starts a new lxd container on machine 4
- 'ssh:user@10.10.0.3' - manually provisions a machine with ssh
+ 'ssh:user@10.10.0.3:/path/to/private/key' - manually provision
+ a machine with ssh and the private key used for authentication
'zone=us-east-1a' - starts a machine in zone us-east-1s on AWS
'maas2.name' - acquire machine maas2.name on MAAS
@@ -929,12 +1000,25 @@
"""
params = client.AddMachineParams()
- params.jobs = ['JobHostUnits']
if spec:
- placement = parse_placement(spec)
- if placement:
- params.placement = placement[0]
+ if spec.startswith("ssh:"):
+ placement, target, private_key_path = spec.split(":")
+ user, host = target.split("@")
+
+ sshProvisioner = provisioner.SSHProvisioner(
+ host=host,
+ user=user,
+ private_key_path=private_key_path,
+ )
+
+ params = sshProvisioner.provision_machine()
+ else:
+ placement = parse_placement(spec)
+ if placement:
+ params.placement = placement[0]
+
+ params.jobs = ['JobHostUnits']
if constraints:
params.constraints = client.Value.from_json(constraints)
@@ -953,6 +1037,17 @@
if error:
raise ValueError("Error adding machine: %s" % error.message)
machine_id = results.machines[0].machine
+
+ if spec:
+ if spec.startswith("ssh:"):
+ # Need to run this after AddMachines has been called,
+ # as we need the machine_id
+ await sshProvisioner.install_agent(
+ self.connection(),
+ params.nonce,
+ machine_id,
+ )
+
log.debug('Added new machine %s', machine_id)
return await self._wait_for_new('machine', machine_id)
@@ -963,7 +1058,8 @@
:param str relation2: '<application>[:<relation_name>]'
"""
- app_facade = client.ApplicationFacade.from_connection(self.connection())
+ connection = self.connection()
+ app_facade = client.ApplicationFacade.from_connection(connection)
log.debug(
'Adding relation %s <-> %s', relation1, relation2)
@@ -1312,7 +1408,8 @@
"""Destroy units by name.
"""
- app_facade = client.ApplicationFacade.from_connection(self.connection())
+ connection = self.connection()
+ app_facade = client.ApplicationFacade.from_connection(connection)
log.debug(
'Destroying unit%s %s',
@@ -1365,11 +1462,28 @@
config[key] = ConfigValue.from_json(value)
return config
- def get_constraints(self):
+ async def get_constraints(self):
"""Return the machine constraints for this model.
+ :returns: A ``dict`` of constraints.
"""
- raise NotImplementedError()
+ constraints = {}
+ client_facade = client.ClientFacade.from_connection(self.connection())
+ result = await client_facade.GetModelConstraints()
+
+ # GetModelConstraints returns GetConstraintsResults which has a 'constraints'
+ # attribute. If no constraints have been set GetConstraintsResults.constraints
+ # is None. Otherwise GetConstraintsResults.constraints has an attribute for each
+ # possible constraint, each of these in turn will be None if they have not been
+ # set.
+ if result.constraints:
+ constraint_types = [a for a in dir(result.constraints)
+ if a in Value._toSchema.keys()]
+ for constraint in constraint_types:
+ value = getattr(result.constraints, constraint)
+ if value is not None:
+ constraints[constraint] = getattr(result.constraints, constraint)
+ return constraints
def import_ssh_key(self, identity):
"""Add a public SSH key from a trusted indentity source to this model.
@@ -1528,31 +1642,79 @@
config[key] = value.value
await config_facade.ModelSet(config)
- def set_constraints(self, constraints):
+ async def set_constraints(self, constraints):
"""Set machine constraints on this model.
- :param :class:`juju.Constraints` constraints: Machine constraints
-
+ :param dict config: Mapping of model constraints
"""
- raise NotImplementedError()
+ client_facade = client.ClientFacade.from_connection(self.connection())
+ await client_facade.SetModelConstraints(
+ application='',
+ constraints=constraints)
- def get_action_output(self, action_uuid, wait=-1):
+ async def get_action_output(self, action_uuid, wait=None):
"""Get the results of an action by ID.
:param str action_uuid: Id of the action
- :param int wait: Time in seconds to wait for action to complete
-
+ :param int wait: Time in seconds to wait for action to complete.
+ :return dict: Output from action
+ :raises: :class:`JujuError` if invalid action_uuid
"""
- raise NotImplementedError()
+ action_facade = client.ActionFacade.from_connection(
+ self.connection()
+ )
+ entity = [{'tag': tag.action(action_uuid)}]
+ # Cannot use self.wait_for_action as the action event has probably
+ # already happened and self.wait_for_action works by processing
+ # model deltas and checking if they match our type. If the action
+ # has already occured then the delta has gone.
- def get_action_status(self, uuid_or_prefix=None, name=None):
- """Get the status of all actions, filtered by ID, ID prefix, or action name.
+ async def _wait_for_action_status():
+ while True:
+ action_output = await action_facade.Actions(entity)
+ if action_output.results[0].status in ('completed', 'failed'):
+ return
+ else:
+ await asyncio.sleep(1)
+ await asyncio.wait_for(
+ _wait_for_action_status(),
+ timeout=wait)
+ action_output = await action_facade.Actions(entity)
+ # ActionResult.output is None if the action produced no output
+ if action_output.results[0].output is None:
+ output = {}
+ else:
+ output = action_output.results[0].output
+ return output
+
+ async def get_action_status(self, uuid_or_prefix=None, name=None):
+ """Get the status of all actions, filtered by ID, ID prefix, or name.
:param str uuid_or_prefix: Filter by action uuid or prefix
:param str name: Filter by action name
"""
- raise NotImplementedError()
+ results = {}
+ action_results = []
+ action_facade = client.ActionFacade.from_connection(
+ self.connection()
+ )
+ if name:
+ name_results = await action_facade.FindActionsByNames([name])
+ action_results.extend(name_results.actions[0].actions)
+ if uuid_or_prefix:
+ # Collect list of actions matching uuid or prefix
+ matching_actions = await action_facade.FindActionTagsByPrefix(
+ [uuid_or_prefix])
+ entities = []
+ for actions in matching_actions.matches.values():
+ entities = [{'tag': a.tag} for a in actions]
+ # Get action results matching action tags
+ uuid_results = await action_facade.Actions(entities)
+ action_results.extend(uuid_results.results)
+ for a in action_results:
+ results[tag.untag('action-', a.action.tag)] = a.status
+ return results
def get_budget(self, budget_name):
"""Get budget usage info.
@@ -1774,6 +1936,9 @@
self.plan = await self.client_facade.GetBundleChanges(
yaml.dump(self.bundle))
+ if self.plan.errors:
+ raise JujuError(self.plan.errors)
+
async def execute_plan(self):
for step in self.plan.changes:
method = getattr(self, step.method)
@@ -1839,7 +2004,11 @@
# Fix up values, as necessary.
if 'parent_id' in params:
- params['parent_id'] = self.resolve(params['parent_id'])
+ if params['parent_id'].startswith('$addUnit'):
+ unit = self.resolve(params['parent_id'])[0]
+ params['parent_id'] = unit.machine.entity_id
+ else:
+ params['parent_id'] = self.resolve(params['parent_id'])
params['constraints'] = parse_constraints(
params.get('constraints'))
diff --git a/modules/libjuju/juju/provisioner.py b/modules/libjuju/juju/provisioner.py
new file mode 100644
index 0000000..91747a4
--- /dev/null
+++ b/modules/libjuju/juju/provisioner.py
@@ -0,0 +1,365 @@
+from .client import client
+
+import paramiko
+import os
+import re
+import tempfile
+import shlex
+from subprocess import (
+ CalledProcessError,
+)
+import uuid
+
+
+arches = [
+ [re.compile("amd64|x86_64"), "amd64"],
+ [re.compile("i?[3-9]86"), "i386"],
+ [re.compile("(arm$)|(armv.*)"), "armhf"],
+ [re.compile("aarch64"), "arm64"],
+ [re.compile("ppc64|ppc64el|ppc64le"), "ppc64el"],
+ [re.compile("ppc64|ppc64el|ppc64le"), "s390x"],
+
+]
+
+
+def normalize_arch(rawArch):
+ """Normalize the architecture string."""
+ for arch in arches:
+ if arch[0].match(rawArch):
+ return arch[1]
+
+
+DETECTION_SCRIPT = """#!/bin/bash
+set -e
+os_id=$(grep '^ID=' /etc/os-release | tr -d '"' | cut -d= -f2)
+if [ "$os_id" = 'centos' ]; then
+ os_version=$(grep '^VERSION_ID=' /etc/os-release | tr -d '"' | cut -d= -f2)
+ echo "centos$os_version"
+else
+ lsb_release -cs
+fi
+uname -m
+grep MemTotal /proc/meminfo
+cat /proc/cpuinfo
+"""
+
+INITIALIZE_UBUNTU_SCRIPT = """set -e
+(id ubuntu &> /dev/null) || useradd -m ubuntu -s /bin/bash
+umask 0077
+temp=$(mktemp)
+echo 'ubuntu ALL=(ALL) NOPASSWD:ALL' > $temp
+install -m 0440 $temp /etc/sudoers.d/90-juju-ubuntu
+rm $temp
+su ubuntu -c 'install -D -m 0600 /dev/null ~/.ssh/authorized_keys'
+export authorized_keys="{}"
+if [ ! -z "$authorized_keys" ]; then
+ su ubuntu -c 'echo $authorized_keys >> ~/.ssh/authorized_keys'
+fi
+"""
+
+
+class SSHProvisioner:
+ """Provision a manually created machine via SSH."""
+ user = ""
+ host = ""
+ private_key_path = ""
+
+ def __init__(self, user, host, private_key_path):
+ self.host = host
+ self.user = user
+ self.private_key_path = private_key_path
+
+ def _get_ssh_client(self, host, user, key):
+ """Return a connected Paramiko ssh object.
+
+ :param str host: The host to connect to.
+ :param str user: The user to connect as.
+ :param str key: The private key to authenticate with.
+
+ :return: object: A paramiko.SSHClient
+ :raises: :class:`paramiko.ssh_exception.SSHException` if the
+ connection failed
+ """
+
+ ssh = paramiko.SSHClient()
+ ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+ pkey = None
+
+ # Read the private key into a paramiko.RSAKey
+ if os.path.exists(key):
+ with open(key, 'r') as f:
+ pkey = paramiko.RSAKey.from_private_key(f)
+
+ #######################################################################
+ # There is a bug in some versions of OpenSSH 4.3 (CentOS/RHEL5) where #
+ # the server may not send the SSH_MSG_USERAUTH_BANNER message except #
+ # when responding to an auth_none request. For example, paramiko will #
+ # attempt to use password authentication when a password is set, but #
+ # the server could deny that, instead requesting keyboard-interactive.#
+ # The hack to workaround this is to attempt a reconnect, which will #
+ # receive the right banner, and authentication can proceed. See the #
+ # following for more info: #
+ # https://github.com/paramiko/paramiko/issues/432 #
+ # https://github.com/paramiko/paramiko/pull/438 #
+ #######################################################################
+
+ try:
+ ssh.connect(host, port=22, username=user, pkey=pkey)
+ except paramiko.ssh_exception.SSHException as e:
+ if 'Error reading SSH protocol banner' == str(e):
+ # Once more, with feeling
+ ssh.connect(host, port=22, username=user, pkey=pkey)
+ else:
+ # Reraise the original exception
+ raise e
+
+ return ssh
+
+ def _run_command(self, ssh, cmd, pty=True):
+ """Run a command remotely via SSH.
+
+ :param object ssh: The SSHClient
+ :param str cmd: The command to execute
+ :param list cmd: The `shlex.split` command to execute
+ :param bool pty: Whether to allocate a pty
+
+ :return: tuple: The stdout and stderr of the command execution
+ :raises: :class:`CalledProcessError` if the command fails
+ """
+
+ if isinstance(cmd, str):
+ cmd = shlex.split(cmd)
+
+ if type(cmd) is not list:
+ cmd = [cmd]
+
+ cmds = ' '.join(cmd)
+ stdin, stdout, stderr = ssh.exec_command(cmds, get_pty=pty)
+ retcode = stdout.channel.recv_exit_status()
+
+ if retcode > 0:
+ output = stderr.read().strip()
+ raise CalledProcessError(returncode=retcode, cmd=cmd,
+ output=output)
+ return (
+ stdout.read().decode('utf-8').strip(),
+ stderr.read().decode('utf-8').strip()
+ )
+
+ def _init_ubuntu_user(self):
+ """Initialize the ubuntu user.
+
+ :return: bool: If the initialization was successful
+ :raises: :class:`paramiko.ssh_exception.AuthenticationException`
+ if the authentication fails
+ """
+
+ # TODO: Test this on an image without the ubuntu user setup.
+
+ auth_user = self.user
+ try:
+ # Run w/o allocating a pty, so we fail if sudo prompts for a passwd
+ ssh = self._get_ssh_client(
+ self.host,
+ "ubuntu",
+ self.private_key_path,
+ )
+
+ stdout, stderr = self._run_command(ssh, "sudo -n true", pty=False)
+ except paramiko.ssh_exception.AuthenticationException as e:
+ raise e
+ else:
+ auth_user = "ubuntu"
+ finally:
+ if ssh:
+ ssh.close()
+
+ # if the above fails, run the init script as the authenticated user
+
+ # Infer the public key
+ public_key = None
+ public_key_path = "{}.pub".format(self.private_key_path)
+
+ if not os.path.exists(public_key_path):
+ raise FileNotFoundError(
+ "Public key '{}' doesn't exist.".format(public_key_path)
+ )
+
+ with open(public_key_path, "r") as f:
+ public_key = f.readline()
+
+ script = INITIALIZE_UBUNTU_SCRIPT.format(public_key)
+
+ try:
+ ssh = self._get_ssh_client(
+ self.host,
+ auth_user,
+ self.private_key_path,
+ )
+
+ self._run_command(
+ ssh,
+ ["sudo", "/bin/bash -c " + shlex.quote(script)],
+ pty=True
+ )
+ except paramiko.ssh_exception.AuthenticationException as e:
+ raise e
+ finally:
+ ssh.close()
+
+ return True
+
+ def _detect_hardware_and_os(self, ssh):
+ """Detect the target hardware capabilities and OS series.
+
+ :param object ssh: The SSHClient
+ :return: str: A raw string containing OS and hardware information.
+ """
+
+ info = {
+ 'series': '',
+ 'arch': '',
+ 'cpu-cores': '',
+ 'mem': '',
+ }
+
+ stdout, stderr = self._run_command(
+ ssh,
+ ["sudo", "/bin/bash -c " + shlex.quote(DETECTION_SCRIPT)],
+ pty=True,
+ )
+
+ lines = stdout.split("\n")
+ info['series'] = lines[0].strip()
+ info['arch'] = normalize_arch(lines[1].strip())
+
+ memKb = re.split('\s+', lines[2])[1]
+
+ # Convert megabytes -> kilobytes
+ info['mem'] = round(int(memKb) / 1024)
+
+ # Detect available CPUs
+ recorded = {}
+ for line in lines[3:]:
+ physical_id = ""
+ print(line)
+
+ if line.find("physical id") == 0:
+ physical_id = line.split(":")[1].strip()
+ elif line.find("cpu cores") == 0:
+ cores = line.split(":")[1].strip()
+
+ if physical_id not in recorded.keys():
+ info['cpu-cores'] += cores
+ recorded[physical_id] = True
+
+ return info
+
+ def provision_machine(self):
+ """Perform the initial provisioning of the target machine.
+
+ :return: bool: The client.AddMachineParams
+ :raises: :class:`paramiko.ssh_exception.AuthenticationException`
+ if the upload fails
+ """
+ params = client.AddMachineParams()
+
+ if self._init_ubuntu_user():
+ try:
+
+ ssh = self._get_ssh_client(
+ self.host,
+ self.user,
+ self.private_key_path
+ )
+
+ hw = self._detect_hardware_and_os(ssh)
+ params.series = hw['series']
+ params.instance_id = "manual:{}".format(self.host)
+ params.nonce = "manual:{}:{}".format(
+ self.host,
+ str(uuid.uuid4()), # a nop for Juju w/manual machines
+ )
+ params.hardware_characteristics = {
+ 'arch': hw['arch'],
+ 'mem': int(hw['mem']),
+ 'cpu-cores': int(hw['cpu-cores']),
+ }
+ params.addresses = [{
+ 'value': self.host,
+ 'type': 'ipv4',
+ 'scope': 'public',
+ }]
+
+ except paramiko.ssh_exception.AuthenticationException as e:
+ raise e
+ finally:
+ ssh.close()
+
+ return params
+
+ async def install_agent(self, connection, nonce, machine_id):
+ """
+ :param object connection: Connection to Juju API
+ :param str nonce: The nonce machine specification
+ :param str machine_id: The id assigned to the machine
+
+ :return: bool: If the initialization was successful
+ """
+
+ # The path where the Juju agent should be installed.
+ data_dir = "/var/lib/juju"
+
+ # Disabling this prevents `apt-get update` from running initially, so
+ # charms will fail to deploy
+ disable_package_commands = False
+
+ client_facade = client.ClientFacade.from_connection(connection)
+ results = await client_facade.ProvisioningScript(
+ data_dir,
+ disable_package_commands,
+ machine_id,
+ nonce,
+ )
+
+ self._run_configure_script(results.script)
+
+ def _run_configure_script(self, script):
+ """Run the script to install the Juju agent on the target machine.
+
+ :param str script: The script returned by the ProvisioningScript API
+ :raises: :class:`paramiko.ssh_exception.AuthenticationException`
+ if the upload fails
+ """
+
+ _, tmpFile = tempfile.mkstemp()
+ with open(tmpFile, 'w') as f:
+ f.write(script)
+
+ try:
+ # get ssh client
+ ssh = self._get_ssh_client(
+ self.host,
+ "ubuntu",
+ self.private_key_path,
+ )
+
+ # copy the local copy of the script to the remote machine
+ sftp = paramiko.SFTPClient.from_transport(ssh.get_transport())
+ sftp.put(
+ tmpFile,
+ tmpFile,
+ )
+
+ # run the provisioning script
+ stdout, stderr = self._run_command(
+ ssh,
+ "sudo /bin/bash {}".format(tmpFile),
+ )
+
+ except paramiko.ssh_exception.AuthenticationException as e:
+ raise e
+ finally:
+ os.remove(tmpFile)
+ ssh.close()
diff --git a/modules/libjuju/juju/tag.py b/modules/libjuju/juju/tag.py
index 319e8f8..282e0a6 100644
--- a/modules/libjuju/juju/tag.py
+++ b/modules/libjuju/juju/tag.py
@@ -34,3 +34,7 @@
def application(app_name):
return _prefix('application-', app_name)
+
+
+def action(action_uuid):
+ return _prefix('action-', action_uuid)
diff --git a/modules/libjuju/juju/unit.py b/modules/libjuju/juju/unit.py
index ce33b08..3be27f2 100644
--- a/modules/libjuju/juju/unit.py
+++ b/modules/libjuju/juju/unit.py
@@ -122,7 +122,7 @@
"""Run command on this unit.
:param str command: The command to run
- :param int timeout: Time to wait before command is considered failed
+ :param int timeout: Time, in seconds, to wait before command is considered failed
:returns: A :class:`juju.action.Action` instance.
"""
@@ -131,6 +131,10 @@
log.debug(
'Running `%s` on %s', command, self.name)
+ if timeout:
+ # Convert seconds to nanoseconds
+ timeout = int(timeout * 1000000000)
+
res = await action.Run(
[],
command,