Add support for local charms

- Adds support for deploying local charms.
- Adds support for using local charm paths in bundles.

diff --git a/docs/narrative/application.rst b/docs/narrative/application.rst
index 01d5bc0..148f1fa 100644 (file)
--- a/docs/narrative/application.rst
+++ b/docs/narrative/application.rst
@@ -32,6 +32,30 @@ To deploy a new application, connect a model and then call its
   )
 
 
   )
 
 

+Deploying a Local Charm
+-----------------------
+To deploy a local charm, first upload it to the model, then
+deploy it using the returned charm url.
+
+.. code:: python
+
+  from juju.model import Model
+
+  model = Model()
+  await model.connect_current()
+
+  # Upload local charm to the model.
+  # The returned 'local:' url can be used to deploy the charm.
+  charm_url = await model.add_local_charm_dir(
+      '/home/tvansteenburgh/src/charms/ubuntu', 'trusty')
+
+  # Deploy the charm using the 'local:' charm.
+  await model.deploy(
+      charm_url,
+      application_name='ubuntu',
+  )
+
+

 Adding Units
 ------------
 To add units to a deployed application, use the
 Adding Units
 ------------
 To add units to a deployed application, use the

diff --git a/examples/localcharm.py b/examples/localcharm.py
new file mode 100644 (file)
index 0000000..bc92914
--- /dev/null
+++ b/examples/localcharm.py
@@ -0,0 +1,40 @@
+"""
+This example shows how to deploy a local charm. It:
+
+1. Connects to current model.
+2. Uploads a local charm (directory on filesystem) to the model.
+3. Deploys the uploaded charm.
+
+"""
+import asyncio
+import logging
+
+from juju.model import Model
+
+
+async def run():
+    model = Model()
+    await model.connect_current()
+
+    # Upload local charm to the model.
+    # The returned 'local:' url can be used to deploy the charm.
+    charm_url = await model.add_local_charm_dir(
+        '/home/tvansteenburgh/src/charms/ubuntu', 'trusty')
+
+    # Deploy the charm using the 'local:' charm.
+    await model.deploy(
+        charm_url,
+        application_name='ubuntu',
+    )
+
+    await model.disconnect()
+    model.loop.stop()
+
+
+logging.basicConfig(level=logging.DEBUG)
+ws_logger = logging.getLogger('websockets.protocol')
+ws_logger.setLevel(logging.INFO)
+loop = asyncio.get_event_loop()
+loop.set_debug(False)
+loop.create_task(run())
+loop.run_forever()

diff --git a/juju/client/connection.py b/juju/client/connection.py
index 5ed073f..9e8cb8f 100644 (file)
--- a/juju/client/connection.py
+++ b/juju/client/connection.py
@@ -9,9 +9,11 @@ import ssl
 import string
 import subprocess
 import websockets
 import string
 import subprocess
 import websockets

+from http.client import HTTPSConnection

 
 import yaml
 
 
 import yaml
 

+from juju import tag

 from juju.errors import JujuAPIError, JujuConnectionError
 
 log = logging.getLogger("websocket")
 from juju.errors import JujuAPIError, JujuConnectionError
 
 log = logging.getLogger("websocket")


@@ -93,6 +95,52 @@ class Connection:
             raise JujuAPIError(result)
         return result
 
             raise JujuAPIError(result)
         return result
 

+    def http_headers(self):
+        """Return dictionary of http headers necessary for making an http
+        connection to the endpoint of this Connection.
+
+        :return: Dictionary of headers
+
+        """
+        if not self.username:
+            return {}
+
+        creds = u'{}:{}'.format(
+            tag.user(self.username),
+            self.password or ''
+        )
+        token = base64.b64encode(creds.encode())
+        return {
+            'Authorization': 'Basic {}'.format(token.decode())
+        }
+
+    def https_connection(self):
+        """Return an https connection to this Connection's endpoint.
+
+        Returns a 3-tuple containing::
+
+            1. The :class:`HTTPSConnection` instance
+            2. Dictionary of auth headers to be used with the connection
+            3. The root url path (str) to be used for requests.
+
+        """
+        endpoint = self.endpoint
+        host, remainder = endpoint.split(':', 1)
+        port = remainder
+        if '/' in remainder:
+            port, _ = remainder.split('/', 1)
+
+        conn = HTTPSConnection(
+            host, int(port),
+            context=self._get_ssl(self.cacert),
+        )
+
+        path = (
+            "/model/{}".format(self.uuid)
+            if self.uuid else ""
+        )
+        return conn, self.http_headers(), path
+

     async def clone(self):
         """Return a new Connection, connected to the same websocket endpoint
         as this one.
     async def clone(self):
         """Return a new Connection, connected to the same websocket endpoint
         as this one.

diff --git a/juju/model.py b/juju/model.py
index c6c01f6..e3a6fea 100644 (file)
--- a/juju/model.py
+++ b/juju/model.py
@@ -1,9 +1,13 @@
 import asyncio
 import collections
 import asyncio
 import collections

+import json

 import logging
 import os
 import re
 import logging
 import os
 import re

+import stat
+import tempfile

 import weakref
 import weakref

+import zipfile

 from concurrent.futures import CancelledError
 from functools import partial
 from pathlib import Path
 from concurrent.futures import CancelledError
 from functools import partial
 from pathlib import Path


@@ -419,6 +423,59 @@ class Model(object):
             await self.connection.close()
             self.connection = None
 
             await self.connection.close()
             self.connection = None
 

+    async def add_local_charm_dir(self, charm_dir, series):
+        """Upload a local charm to the model.
+
+        This will automatically generate an archive from
+        the charm dir.
+
+        :param charm_dir: Path to the charm directory
+        :param series: Charm series
+
+        """
+        fh = tempfile.NamedTemporaryFile()
+        CharmArchiveGenerator(charm_dir).make_archive(fh.name)
+        with fh:
+            func = partial(
+                self.add_local_charm, fh, series, os.stat(fh.name).st_size)
+            charm_url = await self.loop.run_in_executor(None, func)
+
+        log.debug('Uploaded local charm: %s -> %s', charm_dir, charm_url)
+        return charm_url
+
+    def add_local_charm(self, charm_file, series, size=None):
+        """Upload a local charm archive to the model.
+
+        Returns the 'local:...' url that should be used to deploy the charm.
+
+        :param charm_file: Path to charm zip archive
+        :param series: Charm series
+        :param size: Size of the archive, in bytes
+        :return str: 'local:...' url for deploying the charm
+        :raises: :class:`JujuError` if the upload fails
+
+        Uses an https endpoint at the same host:port as the wss.
+        Supports large file uploads.
+
+        .. warning::
+
+           This method will block. Consider using :meth:`add_local_charm_dir`
+           instead.
+
+        """
+        conn, headers, path_prefix = self.connection.https_connection()
+        path = "%s/charms?series=%s" % (path_prefix, series)
+        headers['Content-Type'] = 'application/zip'
+        if size:
+            headers['Content-Length'] = size
+        conn.request("POST", path, charm_file, headers)
+        response = conn.getresponse()
+        result = response.read().decode()
+        if not response.status == 200:
+            raise JujuError(result)
+        result = json.loads(result)
+        return result['charm-url']
+

     def all_units_idle(self):
         """Return True if all units are idle.
 
     def all_units_idle(self):
         """Return True if all units are idle.
 


@@ -882,8 +939,10 @@ class Model(object):
                 for k, v in storage.items()
             }
 
                 for k, v in storage.items()
             }
 

-        is_local = not entity_url.startswith('cs:') and \
+        is_local = (
+            entity_url.startswith('local:') or

             os.path.isdir(entity_url)
             os.path.isdir(entity_url)

+        )

         entity_id = await self.charmstore.entityId(entity_url) \
             if not is_local else entity_url
 
         entity_id = await self.charmstore.entityId(entity_url) \
             if not is_local else entity_url
 


@@ -916,7 +975,8 @@ class Model(object):
             log.debug(
                 'Deploying %s', entity_id)
 
             log.debug(
                 'Deploying %s', entity_id)
 

-            await client_facade.AddCharm(channel, entity_id)
+            if not is_local:
+                await client_facade.AddCharm(channel, entity_id)

             app = client.ApplicationDeploy(
                 application=application_name,
                 channel=channel,
             app = client.ApplicationDeploy(
                 application=application_name,
                 channel=channel,


@@ -1303,6 +1363,21 @@ class Model(object):
         return metrics
 
 
         return metrics
 
 

+def get_charm_series(path):
+    """Inspects the charm directory at ``path`` and returns a default
+    series from its metadata.yaml (the first item in the 'series' list).
+
+    Returns None if no series can be determined.
+
+    """
+    md = Path(path) / "metadata.yaml"
+    if not md.exists():
+        return None
+    data = yaml.load(md.open())
+    series = data.get('series')
+    return series[0] if series else None
+
+

 class BundleHandler(object):
     """
     Handle bundles by using the API to translate bundle YAML into a plan of
 class BundleHandler(object):
     """
     Handle bundles by using the API to translate bundle YAML into a plan of


@@ -1324,6 +1399,54 @@ class BundleHandler(object):
         self.ann_facade = client.AnnotationsFacade()
         self.ann_facade.connect(model.connection)
 
         self.ann_facade = client.AnnotationsFacade()
         self.ann_facade.connect(model.connection)
 

+    async def _handle_local_charms(self, bundle):
+        """Search for references to local charms (i.e. filesystem paths)
+        in the bundle. Upload the local charms to the model, and replace
+        the filesystem paths with appropriate 'local:' paths in the bundle.
+
+        Return the modified bundle.
+
+        :param dict bundle: Bundle dictionary
+        :return: Modified bundle dictionary
+
+        """
+        apps, args = [], []
+
+        default_series = bundle.get('series')
+        for app_name in self.applications:
+            app_dict = bundle['services'][app_name]
+            charm_dir = os.path.abspath(os.path.expanduser(app_dict['charm']))
+            if not os.path.isdir(charm_dir):
+                continue
+            series = (
+                app_dict.get('series') or
+                default_series or
+                get_charm_series(charm_dir)
+            )
+            if not series:
+                raise JujuError(
+                    "Couldn't determine series for charm at {}. "
+                    "Add a 'series' key to the bundle.".format(charm_dir))
+
+            # Keep track of what we need to update. We keep a list of apps
+            # that need to be updated, and a corresponding list of args
+            # needed to update those apps.
+            apps.append(app_name)
+            args.append((charm_dir, series))
+
+        if apps:
+            # If we have apps to update, spawn all the coroutines concurrently
+            # and wait for them to finish.
+            charm_urls = await asyncio.gather(*[
+                asyncio.ensure_future(self.model.add_local_charm_dir(*params))
+                for params in args
+            ])
+            # Update the 'charm:' entry for each app with the new 'local:' url.
+            for app_name, charm_url in zip(apps, charm_urls):
+                bundle['services'][app_name]['charm'] = charm_url
+
+        return bundle
+

     async def fetch_plan(self, entity_id):
         is_local = not entity_id.startswith('cs:') and os.path.isdir(entity_id)
         if is_local:
     async def fetch_plan(self, entity_id):
         is_local = not entity_id.startswith('cs:') and os.path.isdir(entity_id)
         if is_local:


@@ -1333,7 +1456,10 @@ class BundleHandler(object):
                                                       filename='bundle.yaml',
                                                       read_file=True)
         self.bundle = yaml.safe_load(bundle_yaml)
                                                       filename='bundle.yaml',
                                                       read_file=True)
         self.bundle = yaml.safe_load(bundle_yaml)

-        self.plan = await self.client_facade.GetBundleChanges(bundle_yaml)
+        self.bundle = await self._handle_local_charms(self.bundle)
+
+        self.plan = await self.client_facade.GetBundleChanges(
+            yaml.dump(self.bundle))

 
         if self.plan.errors:
             raise JujuError('\n'.join(self.plan.errors))
 
         if self.plan.errors:
             raise JujuError('\n'.join(self.plan.errors))


@@ -1362,6 +1488,11 @@ class BundleHandler(object):
             Series holds the series of the charm to be added
             if the charm default is not sufficient.
         """
             Series holds the series of the charm to be added
             if the charm default is not sufficient.
         """

+        # We don't add local charms because they've already been added
+        # by self._handle_local_charms
+        if charm.startswith('local:'):
+            return charm
+

         entity_id = await self.charmstore.entityId(charm)
         log.debug('Adding %s', entity_id)
         await self.client_facade.AddCharm(None, entity_id)
         entity_id = await self.charmstore.entityId(charm)
         log.debug('Adding %s', entity_id)
         await self.client_facade.AddCharm(None, entity_id)


@@ -1571,3 +1702,79 @@ class CharmStore(object):
             setattr(self, name, coro)
             wrapper = coro
         return wrapper
             setattr(self, name, coro)
             wrapper = coro
         return wrapper

+
+
+class CharmArchiveGenerator(object):
+    def __init__(self, path):
+        self.path = os.path.abspath(os.path.expanduser(path))
+
+    def make_archive(self, path):
+        """Create archive of directory and write to ``path``.
+
+        :param path: Path to archive
+
+        Ignored::
+
+            * build/\* - This is used for packing the charm itself and any
+                          similar tasks.
+            * \*/.\*    - Hidden files are all ignored for now.  This will most
+                          likely be changed into a specific ignore list
+                          (.bzr, etc)
+
+        """
+        zf = zipfile.ZipFile(path, 'w', zipfile.ZIP_DEFLATED)
+        for dirpath, dirnames, filenames in os.walk(self.path):
+            relative_path = dirpath[len(self.path) + 1:]
+            if relative_path and not self._ignore(relative_path):
+                zf.write(dirpath, relative_path)
+            for name in filenames:
+                archive_name = os.path.join(relative_path, name)
+                if not self._ignore(archive_name):
+                    real_path = os.path.join(dirpath, name)
+                    self._check_type(real_path)
+                    if os.path.islink(real_path):
+                        self._check_link(real_path)
+                        self._write_symlink(
+                            zf, os.readlink(real_path), archive_name)
+                    else:
+                        zf.write(real_path, archive_name)
+        zf.close()
+        return path
+
+    def _check_type(self, path):
+        """Check the path
+        """
+        s = os.stat(path)
+        if stat.S_ISDIR(s.st_mode) or stat.S_ISREG(s.st_mode):
+            return path
+        raise ValueError("Invalid Charm at % %s" % (
+            path, "Invalid file type for a charm"))
+
+    def _check_link(self, path):
+        link_path = os.readlink(path)
+        if link_path[0] == "/":
+            raise ValueError(
+                "Invalid Charm at %s: %s" % (
+                    path, "Absolute links are invalid"))
+        path_dir = os.path.dirname(path)
+        link_path = os.path.join(path_dir, link_path)
+        if not link_path.startswith(os.path.abspath(self.path)):
+            raise ValueError(
+                "Invalid charm at %s %s" % (
+                    path, "Only internal symlinks are allowed"))
+
+    def _write_symlink(self, zf, link_target, link_path):
+        """Package symlinks with appropriate zipfile metadata."""
+        info = zipfile.ZipInfo()
+        info.filename = link_path
+        info.create_system = 3
+        # Magic code for symlinks / py2/3 compat
+        # 27166663808 = (stat.S_IFLNK | 0755) << 16
+        info.external_attr = 2716663808
+        zf.writestr(info, link_target)
+
+    def _ignore(self, path):
+        if path == "build" or path.startswith("build/"):
+            return True
+        if path.startswith('.'):
+            return True

diff --git a/juju/tag.py b/juju/tag.py
index d36316d..92c54c1 100644 (file)
--- a/juju/tag.py
+++ b/juju/tag.py
@@ -21,3 +21,7 @@ def credential(cloud, user, credential_name):
 
 def model(cloud_name):
     return _prefix('model-', cloud_name)
 
 def model(cloud_name):
     return _prefix('model-', cloud_name)

+
+
+def user(username):
+    return _prefix('user-', username)