From b7e54690262eb1021274aabb8e93188e49508150 Mon Sep 17 00:00:00 2001 From: Tim Van Steenburgh Date: Thu, 5 Jan 2017 15:55:44 -0500 Subject: [PATCH] Add support for local charms - Adds support for deploying local charms. - Adds support for using local charm paths in bundles. --- docs/narrative/application.rst | 24 ++++ examples/localcharm.py | 40 +++++++ juju/client/connection.py | 48 ++++++++ juju/model.py | 213 ++++++++++++++++++++++++++++++++- juju/tag.py | 4 + 5 files changed, 326 insertions(+), 3 deletions(-) create mode 100644 examples/localcharm.py diff --git a/docs/narrative/application.rst b/docs/narrative/application.rst index 01d5bc0..148f1fa 100644 --- a/docs/narrative/application.rst +++ b/docs/narrative/application.rst @@ -32,6 +32,30 @@ To deploy a new application, connect a model and then call its ) +Deploying a Local Charm +----------------------- +To deploy a local charm, first upload it to the model, then +deploy it using the returned charm url. + +.. code:: python + + from juju.model import Model + + model = Model() + await model.connect_current() + + # Upload local charm to the model. + # The returned 'local:' url can be used to deploy the charm. + charm_url = await model.add_local_charm_dir( + '/home/tvansteenburgh/src/charms/ubuntu', 'trusty') + + # Deploy the charm using the 'local:' charm. + await model.deploy( + charm_url, + application_name='ubuntu', + ) + + Adding Units ------------ To add units to a deployed application, use the diff --git a/examples/localcharm.py b/examples/localcharm.py new file mode 100644 index 0000000..bc92914 --- /dev/null +++ b/examples/localcharm.py @@ -0,0 +1,40 @@ +""" +This example shows how to deploy a local charm. It: + +1. Connects to current model. +2. Uploads a local charm (directory on filesystem) to the model. +3. Deploys the uploaded charm. + +""" +import asyncio +import logging + +from juju.model import Model + + +async def run(): + model = Model() + await model.connect_current() + + # Upload local charm to the model. + # The returned 'local:' url can be used to deploy the charm. + charm_url = await model.add_local_charm_dir( + '/home/tvansteenburgh/src/charms/ubuntu', 'trusty') + + # Deploy the charm using the 'local:' charm. + await model.deploy( + charm_url, + application_name='ubuntu', + ) + + await model.disconnect() + model.loop.stop() + + +logging.basicConfig(level=logging.DEBUG) +ws_logger = logging.getLogger('websockets.protocol') +ws_logger.setLevel(logging.INFO) +loop = asyncio.get_event_loop() +loop.set_debug(False) +loop.create_task(run()) +loop.run_forever() diff --git a/juju/client/connection.py b/juju/client/connection.py index 5ed073f..9e8cb8f 100644 --- a/juju/client/connection.py +++ b/juju/client/connection.py @@ -9,9 +9,11 @@ import ssl import string import subprocess import websockets +from http.client import HTTPSConnection import yaml +from juju import tag from juju.errors import JujuAPIError, JujuConnectionError log = logging.getLogger("websocket") @@ -93,6 +95,52 @@ class Connection: raise JujuAPIError(result) return result + def http_headers(self): + """Return dictionary of http headers necessary for making an http + connection to the endpoint of this Connection. + + :return: Dictionary of headers + + """ + if not self.username: + return {} + + creds = u'{}:{}'.format( + tag.user(self.username), + self.password or '' + ) + token = base64.b64encode(creds.encode()) + return { + 'Authorization': 'Basic {}'.format(token.decode()) + } + + def https_connection(self): + """Return an https connection to this Connection's endpoint. + + Returns a 3-tuple containing:: + + 1. The :class:`HTTPSConnection` instance + 2. Dictionary of auth headers to be used with the connection + 3. The root url path (str) to be used for requests. + + """ + endpoint = self.endpoint + host, remainder = endpoint.split(':', 1) + port = remainder + if '/' in remainder: + port, _ = remainder.split('/', 1) + + conn = HTTPSConnection( + host, int(port), + context=self._get_ssl(self.cacert), + ) + + path = ( + "/model/{}".format(self.uuid) + if self.uuid else "" + ) + return conn, self.http_headers(), path + async def clone(self): """Return a new Connection, connected to the same websocket endpoint as this one. diff --git a/juju/model.py b/juju/model.py index c6c01f6..e3a6fea 100644 --- a/juju/model.py +++ b/juju/model.py @@ -1,9 +1,13 @@ import asyncio import collections +import json import logging import os import re +import stat +import tempfile import weakref +import zipfile from concurrent.futures import CancelledError from functools import partial from pathlib import Path @@ -419,6 +423,59 @@ class Model(object): await self.connection.close() self.connection = None + async def add_local_charm_dir(self, charm_dir, series): + """Upload a local charm to the model. + + This will automatically generate an archive from + the charm dir. + + :param charm_dir: Path to the charm directory + :param series: Charm series + + """ + fh = tempfile.NamedTemporaryFile() + CharmArchiveGenerator(charm_dir).make_archive(fh.name) + with fh: + func = partial( + self.add_local_charm, fh, series, os.stat(fh.name).st_size) + charm_url = await self.loop.run_in_executor(None, func) + + log.debug('Uploaded local charm: %s -> %s', charm_dir, charm_url) + return charm_url + + def add_local_charm(self, charm_file, series, size=None): + """Upload a local charm archive to the model. + + Returns the 'local:...' url that should be used to deploy the charm. + + :param charm_file: Path to charm zip archive + :param series: Charm series + :param size: Size of the archive, in bytes + :return str: 'local:...' url for deploying the charm + :raises: :class:`JujuError` if the upload fails + + Uses an https endpoint at the same host:port as the wss. + Supports large file uploads. + + .. warning:: + + This method will block. Consider using :meth:`add_local_charm_dir` + instead. + + """ + conn, headers, path_prefix = self.connection.https_connection() + path = "%s/charms?series=%s" % (path_prefix, series) + headers['Content-Type'] = 'application/zip' + if size: + headers['Content-Length'] = size + conn.request("POST", path, charm_file, headers) + response = conn.getresponse() + result = response.read().decode() + if not response.status == 200: + raise JujuError(result) + result = json.loads(result) + return result['charm-url'] + def all_units_idle(self): """Return True if all units are idle. @@ -882,8 +939,10 @@ class Model(object): for k, v in storage.items() } - is_local = not entity_url.startswith('cs:') and \ + is_local = ( + entity_url.startswith('local:') or os.path.isdir(entity_url) + ) entity_id = await self.charmstore.entityId(entity_url) \ if not is_local else entity_url @@ -916,7 +975,8 @@ class Model(object): log.debug( 'Deploying %s', entity_id) - await client_facade.AddCharm(channel, entity_id) + if not is_local: + await client_facade.AddCharm(channel, entity_id) app = client.ApplicationDeploy( application=application_name, channel=channel, @@ -1303,6 +1363,21 @@ class Model(object): return metrics +def get_charm_series(path): + """Inspects the charm directory at ``path`` and returns a default + series from its metadata.yaml (the first item in the 'series' list). + + Returns None if no series can be determined. + + """ + md = Path(path) / "metadata.yaml" + if not md.exists(): + return None + data = yaml.load(md.open()) + series = data.get('series') + return series[0] if series else None + + class BundleHandler(object): """ Handle bundles by using the API to translate bundle YAML into a plan of @@ -1324,6 +1399,54 @@ class BundleHandler(object): self.ann_facade = client.AnnotationsFacade() self.ann_facade.connect(model.connection) + async def _handle_local_charms(self, bundle): + """Search for references to local charms (i.e. filesystem paths) + in the bundle. Upload the local charms to the model, and replace + the filesystem paths with appropriate 'local:' paths in the bundle. + + Return the modified bundle. + + :param dict bundle: Bundle dictionary + :return: Modified bundle dictionary + + """ + apps, args = [], [] + + default_series = bundle.get('series') + for app_name in self.applications: + app_dict = bundle['services'][app_name] + charm_dir = os.path.abspath(os.path.expanduser(app_dict['charm'])) + if not os.path.isdir(charm_dir): + continue + series = ( + app_dict.get('series') or + default_series or + get_charm_series(charm_dir) + ) + if not series: + raise JujuError( + "Couldn't determine series for charm at {}. " + "Add a 'series' key to the bundle.".format(charm_dir)) + + # Keep track of what we need to update. We keep a list of apps + # that need to be updated, and a corresponding list of args + # needed to update those apps. + apps.append(app_name) + args.append((charm_dir, series)) + + if apps: + # If we have apps to update, spawn all the coroutines concurrently + # and wait for them to finish. + charm_urls = await asyncio.gather(*[ + asyncio.ensure_future(self.model.add_local_charm_dir(*params)) + for params in args + ]) + # Update the 'charm:' entry for each app with the new 'local:' url. + for app_name, charm_url in zip(apps, charm_urls): + bundle['services'][app_name]['charm'] = charm_url + + return bundle + async def fetch_plan(self, entity_id): is_local = not entity_id.startswith('cs:') and os.path.isdir(entity_id) if is_local: @@ -1333,7 +1456,10 @@ class BundleHandler(object): filename='bundle.yaml', read_file=True) self.bundle = yaml.safe_load(bundle_yaml) - self.plan = await self.client_facade.GetBundleChanges(bundle_yaml) + self.bundle = await self._handle_local_charms(self.bundle) + + self.plan = await self.client_facade.GetBundleChanges( + yaml.dump(self.bundle)) if self.plan.errors: raise JujuError('\n'.join(self.plan.errors)) @@ -1362,6 +1488,11 @@ class BundleHandler(object): Series holds the series of the charm to be added if the charm default is not sufficient. """ + # We don't add local charms because they've already been added + # by self._handle_local_charms + if charm.startswith('local:'): + return charm + entity_id = await self.charmstore.entityId(charm) log.debug('Adding %s', entity_id) await self.client_facade.AddCharm(None, entity_id) @@ -1571,3 +1702,79 @@ class CharmStore(object): setattr(self, name, coro) wrapper = coro return wrapper + + +class CharmArchiveGenerator(object): + def __init__(self, path): + self.path = os.path.abspath(os.path.expanduser(path)) + + def make_archive(self, path): + """Create archive of directory and write to ``path``. + + :param path: Path to archive + + Ignored:: + + * build/\* - This is used for packing the charm itself and any + similar tasks. + * \*/.\* - Hidden files are all ignored for now. This will most + likely be changed into a specific ignore list + (.bzr, etc) + + """ + zf = zipfile.ZipFile(path, 'w', zipfile.ZIP_DEFLATED) + for dirpath, dirnames, filenames in os.walk(self.path): + relative_path = dirpath[len(self.path) + 1:] + if relative_path and not self._ignore(relative_path): + zf.write(dirpath, relative_path) + for name in filenames: + archive_name = os.path.join(relative_path, name) + if not self._ignore(archive_name): + real_path = os.path.join(dirpath, name) + self._check_type(real_path) + if os.path.islink(real_path): + self._check_link(real_path) + self._write_symlink( + zf, os.readlink(real_path), archive_name) + else: + zf.write(real_path, archive_name) + zf.close() + return path + + def _check_type(self, path): + """Check the path + """ + s = os.stat(path) + if stat.S_ISDIR(s.st_mode) or stat.S_ISREG(s.st_mode): + return path + raise ValueError("Invalid Charm at % %s" % ( + path, "Invalid file type for a charm")) + + def _check_link(self, path): + link_path = os.readlink(path) + if link_path[0] == "/": + raise ValueError( + "Invalid Charm at %s: %s" % ( + path, "Absolute links are invalid")) + path_dir = os.path.dirname(path) + link_path = os.path.join(path_dir, link_path) + if not link_path.startswith(os.path.abspath(self.path)): + raise ValueError( + "Invalid charm at %s %s" % ( + path, "Only internal symlinks are allowed")) + + def _write_symlink(self, zf, link_target, link_path): + """Package symlinks with appropriate zipfile metadata.""" + info = zipfile.ZipInfo() + info.filename = link_path + info.create_system = 3 + # Magic code for symlinks / py2/3 compat + # 27166663808 = (stat.S_IFLNK | 0755) << 16 + info.external_attr = 2716663808 + zf.writestr(info, link_target) + + def _ignore(self, path): + if path == "build" or path.startswith("build/"): + return True + if path.startswith('.'): + return True diff --git a/juju/tag.py b/juju/tag.py index d36316d..92c54c1 100644 --- a/juju/tag.py +++ b/juju/tag.py @@ -21,3 +21,7 @@ def credential(cloud, user, credential_name): def model(cloud_name): return _prefix('model-', cloud_name) + + +def user(username): + return _prefix('user-', username) -- 2.25.1