From: Adam Israel Date: Thu, 2 Aug 2018 19:32:00 +0000 (-0400) Subject: Improved Primitive support and better testing X-Git-Tag: BUILD_v4.0.1_2^0 X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2FN2VC.git;a=commitdiff_plain;h=refs%2Fchanges%2F94%2F6394%2F1;hp=421c4a23dc5e60db9596b79ea87cdc19cc463e9b Improved Primitive support and better testing This changeset addresses several issues. - Improve primitive support so the status and output of an executed primitive can be retrieved - Merge latest upstream libjuju (required for new primive features) - New testing framework This is the start of a new testing framework with the ability to create and configure LXD containers with SSH, to use while testing proxy charms. - Add support for using ssh keys with proxy charms See Feature 1429. This uses the per-proxy charm/unit ssh keypair Signed-off-by: Adam Israel --- diff --git a/README.md b/README.md index 7f65584..bed6c90 100644 --- a/README.md +++ b/README.md @@ -124,3 +124,15 @@ Traceback (most recent call last): raise RuntimeError('Event loop is closed') RuntimeError: Event loop is closed ``` + +## Modules + +To update the libjuju module: + +Needs to be fully tested: +```bash +git checkout master +git subtree pull --prefix=modules/libjuju/ --squash libjuju master + +git merge --continue +``` diff --git a/modules/libjuju/.gitignore b/modules/libjuju/.gitignore index 866a785..7614b47 100644 --- a/modules/libjuju/.gitignore +++ b/modules/libjuju/.gitignore @@ -10,3 +10,5 @@ __pycache__/ .\#* dist/ dev/ +.pytest_cache +pytestdebug.log diff --git a/modules/libjuju/.travis.yml b/modules/libjuju/.travis.yml index 0e907f0..4389c8e 100644 --- a/modules/libjuju/.travis.yml +++ b/modules/libjuju/.travis.yml @@ -1,30 +1,50 @@ -dist: trusty +dist: xenial sudo: required language: python -python: - - "3.6" -before_script: - - sudo addgroup lxd || true - - sudo usermod -a -G lxd $USER || true - - sudo ln -s /snap/bin/juju /usr/bin/juju - - sudo ln -s /snap/bin/lxc /usr/bin/lxc +cache: pip before_install: - - sudo add-apt-repository -y ppa:jonathonf/python-3.6 - - sudo add-apt-repository ppa:chris-lea/libsodium -y + - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then sudo add-apt-repository -y ppa:jonathonf/python-3.6; fi + - if [[ $TRAVIS_PYTHON_VERSION == '3.7-dev' ]]; then sudo add-apt-repository -y ppa:deadsnakes/ppa; fi - sudo apt-get update -q - sudo apt-get remove -qy lxd lxd-client - sudo apt-get install snapd libsodium-dev -y - - sudo snap install lxd || true +install: + - sudo snap install lxd || true # ignore failures so that unit tests will still run, at least + - sudo sh -c 'echo PATH=/snap/bin:$PATH >> /etc/environment'; + - sudo snap install jq || true + - sudo snap install juju --classic --$JUJU_CHANNEL || true - sudo snap install juju-wait --classic || true -install: pip install tox-travis + - pip install tox-travis env: global: > TEST_AGENTS='{"agents":[{"url":"https://api.staging.jujucharms.com/identity","username":"libjuju-ci@yellow"}],"key":{"private":"88OOCxIHQNguRG7zFg2y2Hx5Ob0SeVKKBRnjyehverc=","public":"fDn20+5FGyN2hYO7z0rFUyoHGUnfrleslUNtoYsjNSs="}}' - matrix: - - JUJU_CHANNEL=stable - - JUJU_CHANNEL=edge + LXD_PASSWORD="password" + PATH="/snap/bin:$PATH" + +matrix: + include: + - python: 3.6 + env: JUJU_CHANNEL=edge + - python: 3.6 + env: JUJU_CHANNEL=stable + - python: 3.7-dev + env: JUJU_CHANNEL=stable + - python: 3.7-dev + env: JUJU_CHANNEL=edge +before_script: + - sudo lxd waitready --timeout 30 + - sudo lxc storage create default dir + - sudo lxd init --auto --trust-password="${LXD_PASSWORD}" --network-address='[::]' --network-port=8443 + + # I suspect these could be integrated into the lxd init call + - sudo sudo mkdir /var/snap/lxd/common/lxd/storage-pools/juju-zfs + - sudo lxc storage create juju-zfs dir source=/var/snap/lxd/common/lxd/storage-pools/juju-zfs # Horrible workaround to LP Bug #1738614 + + - sudo addgroup lxd || true + - sudo usermod -a -G lxd $USER || true + # Trigger generation of the client certificates + - sg lxd -c "echo y|lxc remote add "$USER" https://0.0.0.0:8443 --accept-certificate --password='${LXD_PASSWORD}'" + script: - - sudo snap install juju --classic --$JUJU_CHANNEL - - sudo ln -s /snap/bin/juju /usr/bin/juju || true - - sudo -E sudo -u $USER -E /snap/bin/juju bootstrap localhost test --config 'identity-url=https://api.staging.jujucharms.com/identity' --config 'allow-model-access=true' - - tox -e py35,integration + - sudo -E sudo -u $USER -E juju bootstrap localhost test --config 'identity-url=https://api.staging.jujucharms.com/identity' --config 'allow-model-access=true' + - tox -e py3,integration,serial diff --git a/modules/libjuju/VERSION b/modules/libjuju/VERSION index f38fc53..2cc4a7a 100644 --- a/modules/libjuju/VERSION +++ b/modules/libjuju/VERSION @@ -1 +1,5 @@ +<<<<<<< HEAD 0.7.3 +======= +0.9.1 +>>>>>>> 8a2d5bc35a302a970244b3c307a4f47deac0af63 diff --git a/modules/libjuju/docs/changelog.rst b/modules/libjuju/docs/changelog.rst index caf778e..a4a4222 100644 --- a/modules/libjuju/docs/changelog.rst +++ b/modules/libjuju/docs/changelog.rst @@ -1,6 +1,60 @@ Changelog --------- +<<<<<<< HEAD +======= +0.9.1 +^^^^^ +Monday July 16 2018 + +* Update websockets to 6.0 to fix OS X support due to Brew update to Py3.7 (#254) + + +0.9.0 +^^^^^ +Friday June 29 2018 + +* python3.7 compatibility updates (#251) +* Handle juju not installed in is_bootstrapped for tests (#250) +* Add app.reset_config(list). (#249) +* Implement model.get_action_status (#248) +* Fix `make client` in Python 3.6 (#247) + + +0.8.0 +^^^^^ +Thursday June 14 2018 + +* Add support for adding a manual (ssh) machine (#240) +* Backwards compatibility fixes (#213) +* Implement model.get_action_output (#242) +* Fix JSON serialization error for bundle with lxd to unit placement (#243) +* Fix reference in docs to connect_current (#239) +* Wrap machine agent status workaround in version check (#238) +* Convert seconds to nanoseconds for juju.unit.run (#237) +* Fix spurious intermittent failure in test_machines.py::test_status (#236) +* Define an unused juju-zfs lxd storage pool for Travis (#235) +* Add support for Application get_actions (#234) + + +0.7.5 +^^^^^ +Friday May 18 2018 + +* Surface errors from bundle plan (#233) +* Always send auth-tag even with macaroon auth (#217) +* Inline jsonfile credential when sending to controller (#231) + +0.7.4 +^^^^^ +Tuesday Apr 24 2018 + +* Always parse tags and spaces constraints to lists (#228) +* Doc index improvements (#211) +* Add doc req to force newer pymacaroons to fix RTD builds +* Fix dependency conflict for building docs + +>>>>>>> 8a2d5bc35a302a970244b3c307a4f47deac0af63 0.7.3 ^^^^^ Tuesday Feb 20 2018 @@ -49,6 +103,7 @@ Fri Sept 29 2017 * Make Application.upgrade_charm upgrade resources (#158) * Expand integration tests to use stable/edge versions of juju (#155) * Move docs to ReadTheDocs (https://pythonlibjuju.readthedocs.io/en/latest/) +<<<<<<< HEAD 0.6.1 ^^^^^ @@ -60,6 +115,8 @@ Fri Sept 29 2017 * Make Application.upgrade_charm upgrade resources (#158) * Expand integration tests to use stable/edge versions of juju (#155) * Move docs to ReadTheDocs (https://pythonlibjuju.readthedocs.io/en/latest/) +======= +>>>>>>> 8a2d5bc35a302a970244b3c307a4f47deac0af63 0.6.0 ^^^^^ diff --git a/modules/libjuju/docs/narrative/controller.rst b/modules/libjuju/docs/narrative/controller.rst index 2da0e7b..1d86321 100644 --- a/modules/libjuju/docs/narrative/controller.rst +++ b/modules/libjuju/docs/narrative/controller.rst @@ -6,9 +6,9 @@ these endpoints. Connecting to the controller endpoint is useful if you want to programmatically create a new model. If the model you want to use already exists, you can -connect directly to it (see :doc:`model`). +connect directly to it (see py:doc:`model`). -For api docs, see :class:`juju.controller.Controller`. +For API docs, see py:class:`juju.controller.Controller`. Connecting to the Current Controller @@ -21,7 +21,7 @@ Connect to the currently active Juju controller (the one returned by from juju.controller import Controller controller = Controller() - await controller.connect_current() + await controller.connect() Connecting to a Named Controller @@ -33,68 +33,60 @@ Connect to a controller by name. from juju.controller import Controller controller = Controller() - await controller.connect_controller('mycontroller') + await controller.connect('mycontroller') -Connecting with Username/Password Authentication ------------------------------------------------- -The most flexible, but also most verbose, way to connect is using the API -endpoint url and credentials directly. This method does NOT require the Juju -CLI client to be installed. +Connecting with Authentication +------------------------------ +You can control what user you are connecting with by specifying either a +username/password pair, or a macaroon or bakery client that can provide +a macaroon. -.. code:: python - from juju.controller import Controller +.. code:: python controller = Controller() + await controller.connect(username='admin', + password='f53f08cfc32a2e257fe5393271d89d62') + + # or with a macaroon + await controller.connect(macaroons=[ + { + "Name": "macaroon-218d87053ad19626bcd5a0eef0bc9ba8bd4fbd80a968f52a5fd430b2aa8660df", + "Value": "W3siY2F2ZWF0cyI6 ... jBkZiJ9XQ==", + "Domain": "10.130.48.27", + "Path": "/auth", + "Secure": false, + "HostOnly": true, + "Expires": "2018-03-07T22:07:23Z", + }, + ]) + + # or with a bakery client + from macaroonbakery.httpbakery import Client + from http.cookiejar import FileCookieJar + + bakery_client=Client() + bakery_client.cookies = FileCookieJar('cookies.txt') + controller = Controller() + await controller.connect(bakery_client=bakery_client) + - controller_endpoint = '10.0.4.171:17070' - username = 'admin' - password = 'f53f08cfc32a2e257fe5393271d89d62' - - # Left out for brevity, but if you have a cert string you should pass it in. - # You can copy the cert from the output of The `juju show-controller` - # command. - cacert = None - - await controller.connect( - controller_endpoint, - username, - password, - cacert, - ) - - -Connecting with Macaroon Authentication ---------------------------------------- -To connect to a shared controller, you'll need -to use macaroon authentication. The simplest example is shown below, and uses -already-discharged macaroons from the local filesystem. This will work if you -have the Juju CLI installed. - -.. note:: - The library does not yet contain support for fetching and discharging - macaroons. Until it does, if you want to use macaroon auth, you'll need - to supply already-discharged macaroons yourself. +Connecting with an Explicit Endpoint +------------------------------------ +The most flexible, but also most verbose, way to connect is using the API +endpoint url and credentials directly. This method does NOT require the +Juju CLI client to be installed. .. code:: python - from juju.client.connection import get_macaroons() - from juju.controller import Controller - controller = Controller() - - controller_endpoint = '10.0.4.171:17070' - username = None - password = None - cacert = None - macaroons = get_macaroons() - await controller.connect( - controller_endpoint, - username, - password, - cacert, - macaroons, + endpoint='10.0.4.171:17070', + username='admin', + password='f53f08cfc32a2e257fe5393271d89d62', + cacert=None, # Left out for brevity, but if you have a cert string you + # should pass it in. You can get the cert from the output + # of The `juju show-controller` command. ) diff --git a/modules/libjuju/docs/narrative/model.rst b/modules/libjuju/docs/narrative/model.rst index 57dbc81..42633a1 100644 --- a/modules/libjuju/docs/narrative/model.rst +++ b/modules/libjuju/docs/narrative/model.rst @@ -4,7 +4,7 @@ A Juju controller provides websocket endpoints for each of its models. In order to do anything useful with a model, the juju lib must connect to one of these endpoints. There are several ways to do this. -For api docs, see :class:`juju.model.Model`. +For api docs, see py:class:`juju.model.Model`. Connecting to the Current Model @@ -14,10 +14,8 @@ Connect to the currently active Juju model (the one returned by .. code:: python - from juju.model import Model - model = Model() - await model.connect_current() + await model.connect() Connecting to a Named Model @@ -28,88 +26,74 @@ This only works if you have the Juju CLI client installed. .. code:: python - # $ juju switch - # juju-2.0.1:admin/libjuju - - from juju.model import Model - model = Model() - await model.connect_model('juju-2.0.1:admin/libjuju') + await model.connect('juju-2.0.1:admin/libjuju') -Connecting with Username/Password Authentication ------------------------------------------------- -The most flexible, but also most verbose, way to connect is using the API -endpoint url and credentials directly. This method does NOT require the Juju -CLI client to be installed. +Connecting with Authentication +------------------------------ +You can control what user you are connecting with by specifying either a +username/password pair, or a macaroon or bakery client that can provide +a macaroon. -.. code:: python - from juju.model import Model +.. code:: python model = Model() + await model.connect(username='admin', + password='f53f08cfc32a2e257fe5393271d89d62') + + # or with a macaroon + await model.connect(macaroons=[ + { + "Name": "macaroon-218d87053ad19626bcd5a0eef0bc9ba8bd4fbd80a968f52a5fd430b2aa8660df", + "Value": "W3siY2F2ZWF0cyI6 ... jBkZiJ9XQ==", + "Domain": "10.130.48.27", + "Path": "/auth", + "Secure": false, + "HostOnly": true, + "Expires": "2018-03-07T22:07:23Z", + }, + ]) - controller_endpoint = '10.0.4.171:17070' - model_uuid = 'e8399ac7-078c-4817-8e5e-32316d55b083' - username = 'admin' - password = 'f53f08cfc32a2e257fe5393271d89d62' - - # Left out for brevity, but if you have a cert string you should pass it in. - # You can copy the cert from the output of The `juju show-controller` - # command. - cacert = None - - await model.connect( - controller_endpoint, - model_uuid, - username, - password, - cacert, - ) - + # or with a bakery client + from macaroonbakery.httpbakery import Client + from http.cookiejar import FileCookieJar -Connecting with Macaroon Authentication ---------------------------------------- -To connect to a shared model, or a model an a shared controller, you'll need -to use macaroon authentication. The simplest example is shown below, and uses -already-discharged macaroons from the local filesystem. This will work if you -have the Juju CLI installed. + bakery_client=Client() + bakery_client.cookies = FileCookieJar('cookies.txt') + model = Model() + await model.connect(bakery_client=bakery_client) + -.. note:: - The library does not yet contain support for fetching and discharging - macaroons. Until it does, if you want to use macaroon auth, you'll need - to supply already-discharged macaroons yourself. +Connecting with an Explicit Endpoint +------------------------------------ +The most flexible, but also most verbose, way to connect is using the API +endpoint url, model UUID, and credentials directly. This method does NOT +require the Juju CLI client to be installed. .. code:: python - from juju.client.connection import get_macaroons() from juju.model import Model model = Model() - - controller_endpoint = '10.0.4.171:17070' - model_uuid = 'e8399ac7-078c-4817-8e5e-32316d55b083' - username = None - password = None - cacert = None - macaroons = get_macaroons() - await model.connect( - controller_endpoint, - model_uuid, - username, - password, - cacert, - macaroons, + endpoint='10.0.4.171:17070', + uuid='e8399ac7-078c-4817-8e5e-32316d55b083', + username='admin', + password='f53f08cfc32a2e257fe5393271d89d62', + cacert=None, # Left out for brevity, but if you have a cert string you + # should pass it in. You can get the cert from the output + # of The `juju show-controller` command. ) Creating and Destroying a Model ------------------------------- Example of creating a new model and then destroying it. See -:meth:`juju.controller.Controller.add_model` and -:meth:`juju.controller.Controller.destroy_model` for more info. +py:method:`juju.controller.Controller.add_model` and +py:method:`juju.controller.Controller.destroy_model` for more info. .. code:: python @@ -134,8 +118,8 @@ Example of creating a new model and then destroying it. See Adding Machines and Containers ------------------------------ To add a machine or container, connect to a model and then call its -:meth:`~juju.model.Model.add_machine` method. A -:class:`~juju.machine.Machine` instance is returned. The machine id +py:method:`~juju.model.Model.add_machine` method. A +py:class:`~juju.machine.Machine` instance is returned. The machine id can be used to deploy a charm to a specific machine or container. .. code:: python @@ -192,7 +176,7 @@ Reacting to Changes in a Model ------------------------------ To watch for and respond to changes in a model, register an observer with the model. The easiest way to do this is by creating a -:class:`juju.model.ModelObserver` subclass. +py:class:`juju.model.ModelObserver` subclass. .. code:: python @@ -283,7 +267,7 @@ to the entity and type of change that you wish to handle. # specific handler method is not defined. -Any :class:`juju.model.ModelEntity` object can be observed directly by +Any py:class:`juju.model.ModelEntity` object can be observed directly by registering callbacks on the object itself. .. code:: python diff --git a/modules/libjuju/docs/readme.rst b/modules/libjuju/docs/readme.rst index 886550d..87666d0 100644 --- a/modules/libjuju/docs/readme.rst +++ b/modules/libjuju/docs/readme.rst @@ -38,10 +38,7 @@ Quickstart ---------- Here's a simple example that shows basic usage of the library. The example connects to the currently active Juju model, deploys a single unit of the -ubuntu charm, then exits. - -More examples can be found in the `examples/` directory of the source tree, -and in the documentation. +ubuntu charm, then exits: .. code:: python @@ -95,3 +92,12 @@ and in the documentation. if __name__ == '__main__': main() + + +More examples can be found in the docs, as well as in the ``examples/`` +directory of the source tree which can be run using ``tox``. For +example, to run ``examples/connect_current_model.py``, use: + +.. code:: bash + + tox -e example -- examples/connect_current_model.py diff --git a/modules/libjuju/docs/upstream-updates/index.rst b/modules/libjuju/docs/upstream-updates/index.rst index 7082a6e..41f448a 100644 --- a/modules/libjuju/docs/upstream-updates/index.rst +++ b/modules/libjuju/docs/upstream-updates/index.rst @@ -48,6 +48,54 @@ as well as one or more of the `juju/client/_clientX.py` files, depending on which facades were touched. +Integrating into the Object Layer +--------------------------------- + +Once the raw client APIs are synced, you may need to integrate any new or +changed API calls into the object layer, to provide a clean, Pythonic way +to interact with the model. This may be as simple as adding an optional +parameter to an existing model method, tweaking what manipulations, if any +the model method does to the data before it is sent to the API, or it may +require adding an entirely new model method to capture the new functionality. + +In general, the approach should be to make the interactions with the model +layer use the same patterns as when you use the CLI, just with Python idioms +and OO approaches. + +When trying to determine what client calls need to be made and what data to +be sent for a given Juju CLI action, it is very useful to add +`--debug --logging-config TRACE` to any Juju CLI command to view the full +conversation between the CLI client and the API server. For example: + +``` +[johnsca@murdoch:~] $ juju deploy --debug --logging-config TRACE ./builds/test +11:51:20 INFO juju.cmd supercommand.go:56 running juju [2.3.5 gc go1.10] +11:51:20 DEBUG juju.cmd supercommand.go:57 args: []string{"/snap/juju/3884/bin/juju", "deploy", "--debug", "--logging-config", "TRACE", "./builds/test"} +11:51:20 INFO juju.juju api.go:67 connecting to API addresses: [35.172.119.191:17070 172.31.94.16:17070 252.94.16.1:17070] +11:51:20 TRACE juju.api certpool.go:49 cert dir "/etc/juju/certs.d" does not exist +11:51:20 DEBUG juju.api apiclient.go:843 successfully dialed "wss://35.172.119.191:17070/model/a7317969-6dab-4ba4-844b-af3d661c228d/api" +11:51:20 INFO juju.api apiclient.go:597 connection established to "wss://35.172.119.191:17070/model/a7317969-6dab-4ba4-844b-af3d661c228d/api" +... +11:51:20 INFO juju.cmd.juju.application series_selector.go:71 with the configured model default series "xenial" +11:51:20 DEBUG httpbakery client.go:244 client do POST https://35.172.119.191:17070/model/a7317969-6dab-4ba4-844b-af3d661c228d/charms?revision=0&schema=local&series=xenial { +11:51:21 DEBUG httpbakery client.go:246 } -> error +11:51:21 INFO cmd deploy.go:1096 Deploying charm "local:xenial/test-0". +11:51:21 TRACE juju.rpc.jsoncodec codec.go:225 -> {"request-id":3,"type":"Charms","version":2,"request":"CharmInfo","params":{"url":"local:xenial/test-0"}} +11:51:21 TRACE juju.rpc.jsoncodec codec.go:120 <- {"request-id":3,"response":{"revision":0,"url":"local:xenial/test-0","config":{"test":{"type":"string","default":""}},"meta":{"name":"test","summary":"test","description":"test","subordinate":false,"series":["xenial"],"resources":{"dummy":{"name":"dummy","type":"file","path":"dummy.snap","description":"dummy snap"}},"min-juju-version":"0.0.0"},"actions":{}}} +11:51:21 TRACE juju.rpc.jsoncodec codec.go:225 -> {"request-id":4,"type":"Charms","version":2,"request":"IsMetered","params":{"url":"local:xenial/test-0"}} +11:51:21 TRACE juju.rpc.jsoncodec codec.go:120 <- {"request-id":4,"response":{"metered":false}} +11:51:21 TRACE juju.rpc.jsoncodec codec.go:225 -> {"request-id":5,"type":"Resources","version":1,"request":"AddPendingResources","params":{"tag":"application-test","url":"local:xenial/test-0","channel":"","macaroon":null,"resources":[{"name":"dummy","type":"file","path":"dummy.snap","description":"dummy snap","origin":"store","revision":-1,"fingerprint":"","size":0}]}} +11:51:21 TRACE juju.rpc.jsoncodec codec.go:120 <- {"request-id":5,"response":{"pending-ids":["c0ffdd92-da23-4fb2-8d41-d82d58423447"]}} +11:51:21 TRACE juju.rpc.jsoncodec codec.go:225 -> {"request-id":6,"type":"Application","version":5,"request":"Deploy","params":{"applications":[{"application":"test","series":"xenial","charm-url":"local:xenial/test-0","channel":"","num-units":1,"config-yaml":"","constraints":{},"resources":{"dummy":"c0ffdd92-da23-4fb2-8d41-d82d58423447"}}]}} +11:51:21 TRACE juju.rpc.jsoncodec codec.go:120 <- {"request-id":6,"response":{"results":[{}]}} +11:51:21 TRACE juju.rpc.jsoncodec codec.go:123 <- error: read tcp 192.168.1.102:52168->35.172.119.191:17070: use of closed network connection (closing true) +11:51:21 DEBUG juju.api monitor.go:35 RPC connection died +11:51:21 INFO cmd supercommand.go:465 command finished +``` + +Note that this will contain login information (which has been removed from the above). + + Overrides --------- diff --git a/modules/libjuju/examples/action.py b/modules/libjuju/examples/action.py index 4a3cc6d..f839f11 100644 --- a/modules/libjuju/examples/action.py +++ b/modules/libjuju/examples/action.py @@ -7,7 +7,6 @@ This example: 4. Waits for the action results to come back, then exits. """ -import asyncio import logging from juju import loop @@ -27,8 +26,8 @@ async def run_action(unit): async def main(): model = Model() + # connect to current model with current user, per Juju CLI await model.connect() - await model.reset(force=True) app = await model.deploy( 'git', diff --git a/modules/libjuju/examples/add_model.py b/modules/libjuju/examples/add_model.py index 0e96fa1..88766f1 100644 --- a/modules/libjuju/examples/add_model.py +++ b/modules/libjuju/examples/add_model.py @@ -19,6 +19,7 @@ LOG = getLogger(__name__) async def main(): controller = Controller() print("Connecting to controller") + # connect to current controller with current user, per Juju CLI await controller.connect() try: diff --git a/modules/libjuju/examples/config.py b/modules/libjuju/examples/config.py index bad5b6d..c7580f6 100644 --- a/modules/libjuju/examples/config.py +++ b/modules/libjuju/examples/config.py @@ -6,7 +6,6 @@ This example: 3. Deploys a charm and prints its config and constraints """ -import asyncio import logging from juju.model import Model @@ -19,8 +18,8 @@ MB = 1 async def main(): model = Model() + # connect to current model with current user, per Juju CLI await model.connect() - await model.reset(force=True) ubuntu_app = await model.deploy( 'mysql', @@ -47,7 +46,7 @@ async def main(): await model.disconnect() - + if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG) ws_logger = logging.getLogger('websockets.protocol') diff --git a/modules/libjuju/examples/connect_current_model.py b/modules/libjuju/examples/connect_current_model.py new file mode 100644 index 0000000..b46a09c --- /dev/null +++ b/modules/libjuju/examples/connect_current_model.py @@ -0,0 +1,27 @@ +""" +This is a very basic example that connects to the currently selected model +and prints the number of applications deployed to it. +""" +import logging + +from juju import loop +from juju.model import Model + +log = logging.getLogger(__name__) + + +async def main(): + model = Model() + try: + # connect to the current model with the current user, per the Juju CLI + await model.connect() + print('There are {} applications'.format(len(model.applications))) + finally: + if model.is_connected(): + print('Disconnecting from model') + await model.disconnect() + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + loop.run(main()) diff --git a/modules/libjuju/examples/controller.py b/modules/libjuju/examples/controller.py index 3f029ab..b61a6f6 100644 --- a/modules/libjuju/examples/controller.py +++ b/modules/libjuju/examples/controller.py @@ -8,7 +8,6 @@ This example: 5. Destroys the model """ -import asyncio import logging from juju.controller import Controller @@ -17,6 +16,7 @@ from juju import loop async def main(): controller = Controller() + # connect to current controller with current user, per Juju CLI await controller.connect() model = await controller.add_model( 'my-test-model', diff --git a/modules/libjuju/examples/credential.py b/modules/libjuju/examples/credential.py new file mode 100644 index 0000000..e653536 --- /dev/null +++ b/modules/libjuju/examples/credential.py @@ -0,0 +1,47 @@ +import sys +from juju import loop +from juju.controller import Controller + + +async def main(cloud_name, credential_name): + controller = Controller() + model = None + print('Connecting to controller') + # connect to current controller with current user, per Juju CLI + await controller.connect() + try: + print('Adding model') + model = await controller.add_model( + 'test', + cloud_name=cloud_name, + credential_name=credential_name) + + # verify credential + print("Verify model's credential: {}".format( + model.info.cloud_credential_tag)) + + # verify we can deploy + print('Deploying ubuntu') + app = await model.deploy('ubuntu-10') + + print('Waiting for active') + await model.block_until( + lambda: app.units and all(unit.workload_status == 'active' + for unit in app.units)) + + print('Removing ubuntu') + await app.remove() + finally: + print('Cleaning up') + if model: + print('Removing model') + model_uuid = model.info.uuid + await model.disconnect() + await controller.destroy_model(model_uuid) + print('Disconnecting') + await controller.disconnect() + + +if __name__ == '__main__': + assert len(sys.argv) > 2, 'Please provide a cloud and credential name' + loop.run(main(sys.argv[1], sys.argv[2])) diff --git a/modules/libjuju/examples/deploy.py b/modules/libjuju/examples/deploy.py index b14e4ca..43764d7 100644 --- a/modules/libjuju/examples/deploy.py +++ b/modules/libjuju/examples/deploy.py @@ -13,6 +13,7 @@ from juju.model import Model async def main(): model = Model() print('Connecting to model') + # connect to current model with current user, per Juju CLI await model.connect() try: diff --git a/modules/libjuju/examples/future.py b/modules/libjuju/examples/future.py index c93981a..5e974cf 100644 --- a/modules/libjuju/examples/future.py +++ b/modules/libjuju/examples/future.py @@ -2,7 +2,6 @@ This example doesn't work - it demonstrates features that don't exist yet. """ -import asyncio import logging from juju.model import Model @@ -11,8 +10,8 @@ from juju import loop async def main(): model = Model() + # connect to current model with current user, per Juju CLI await model.connect() - await model.reset(force=True) goal_state = Model.from_yaml('bundle-like-thing') ubuntu_app = await model.deploy( diff --git a/modules/libjuju/examples/livemodel.py b/modules/libjuju/examples/livemodel.py index a15e9f7..1b10ac9 100644 --- a/modules/libjuju/examples/livemodel.py +++ b/modules/libjuju/examples/livemodel.py @@ -6,8 +6,6 @@ This example: 3. Runs forever (kill with Ctrl-C) """ -import asyncio - from juju.model import Model from juju import loop @@ -21,6 +19,7 @@ async def on_model_change(delta, old, new, model): async def watch_model(): model = Model() + # connect to current model with current user, per Juju CLI await model.connect() model.add_observer(on_model_change) diff --git a/modules/libjuju/examples/relate.py b/modules/libjuju/examples/relate.py index c0ce4c6..347e021 100644 --- a/modules/libjuju/examples/relate.py +++ b/modules/libjuju/examples/relate.py @@ -40,6 +40,7 @@ class MyModelObserver(ModelObserver): async def main(): model = Model() + # connect to current model with current user, per Juju CLI await model.connect() try: diff --git a/modules/libjuju/examples/unitrun.py b/modules/libjuju/examples/unitrun.py index b6e2240..805f0ae 100644 --- a/modules/libjuju/examples/unitrun.py +++ b/modules/libjuju/examples/unitrun.py @@ -7,7 +7,6 @@ This example: 4. Waits for the action results to come back, then exits. """ -import asyncio import logging from juju.model import Model @@ -24,8 +23,8 @@ async def run_command(unit): async def main(): model = Model() + # connect to current model with current user, per Juju CLI await model.connect() - await model.reset(force=True) app = await model.deploy( 'ubuntu-0', diff --git a/modules/libjuju/juju/application.py b/modules/libjuju/juju/application.py index 555bb3d..84afebe 100644 --- a/modules/libjuju/juju/application.py +++ b/modules/libjuju/juju/application.py @@ -228,13 +228,24 @@ class Application(model.ModelEntity): result = (await app_facade.Get(self.name)).constraints return vars(result) if result else result - def get_actions(self, schema=False): + async def get_actions(self, schema=False): """Get actions defined for this application. :param bool schema: Return the full action schema - + :return dict: The charms actions, empty dict if none are defined. """ - raise NotImplementedError() + actions = {} + entity = [{"tag": self.tag}] + action_facade = client.ActionFacade.from_connection(self.connection) + results = ( + await action_facade.ApplicationsCharmsActions(entity)).results + for result in results: + if result.application_tag == self.tag and result.actions: + actions = result.actions + break + if not schema: + actions = {k: v['description'] for k, v in actions.items()} + return actions def get_resources(self, details=False): """Return resources for this application. @@ -284,12 +295,10 @@ class Application(model.ModelEntity): ) return await self.ann_facade.Set([ann]) - async def set_config(self, config, to_default=False): + async def set_config(self, config): """Set configuration options for this application. :param config: Dict of configuration to set - :param bool to_default: Set application options to default values - """ app_facade = client.ApplicationFacade.from_connection(self.connection) @@ -298,6 +307,19 @@ class Application(model.ModelEntity): return await app_facade.Set(self.name, config) + async def reset_config(self, to_default): + """ + Restore application config to default values. + + :param list to_default: A list of config options to be reset to their default value. + """ + app_facade = client.ApplicationFacade.from_connection(self.connection) + + log.debug( + 'Restoring default config for %s: %s', self.name, to_default) + + return await app_facade.Unset(self.name, to_default) + async def set_constraints(self, constraints): """Set machine constraints for this application. diff --git a/modules/libjuju/juju/client/connection.py b/modules/libjuju/juju/client/connection.py index bdd1c3f..13770a5 100644 --- a/modules/libjuju/juju/client/connection.py +++ b/modules/libjuju/juju/client/connection.py @@ -109,20 +109,22 @@ class Connection: If uuid is None, the connection will be to the controller. Otherwise it will be to the model. - :param str endpoint The hostname:port of the controller to connect to. - :param str uuid The model UUID to connect to (None for a + + :param str endpoint: The hostname:port of the controller to connect to. + :param str uuid: The model UUID to connect to (None for a controller-only connection). - :param str username The username for controller-local users (or None + :param str username: The username for controller-local users (or None to use macaroon-based login.) - :param str password The password for controller-local users. - :param str cacert The CA certificate of the controller (PEM formatted). - :param httpbakery.Client bakery_client The macaroon bakery client to + :param str password: The password for controller-local users. + :param str cacert: The CA certificate of the controller + (PEM formatted). + :param httpbakery.Client bakery_client: The macaroon bakery client to to use when performing macaroon-based login. Macaroon tokens acquired when logging will be saved to bakery_client.cookies. If this is None, a default bakery_client will be used. - :param loop asyncio.BaseEventLoop The event loop to use for async + :param asyncio.BaseEventLoop loop: The event loop to use for async operations. - :param max_frame_size The maximum websocket frame size to allow. + :param int max_frame_size: The maximum websocket frame size to allow. """ self = cls() if endpoint is None: @@ -519,8 +521,8 @@ class Connection: async def login(self): params = {} + params['auth-tag'] = self.usertag if self.password: - params['auth-tag'] = self.usertag params['credentials'] = self.password else: macaroons = _macaroons_for_domain(self.bakery_client.cookies, diff --git a/modules/libjuju/juju/client/connector.py b/modules/libjuju/juju/client/connector.py index 64fbe44..a30adbf 100644 --- a/modules/libjuju/juju/client/connector.py +++ b/modules/libjuju/juju/client/connector.py @@ -4,6 +4,7 @@ import copy import macaroonbakery.httpbakery as httpbakery from juju.client.connection import Connection +from juju.client.gocookies import go_to_py_cookie, GoCookieJar from juju.client.jujudata import FileJujuData from juju.errors import JujuConnectionError, JujuError @@ -56,6 +57,14 @@ class Connector: kwargs.setdefault('loop', self.loop) kwargs.setdefault('max_frame_size', self.max_frame_size) kwargs.setdefault('bakery_client', self.bakery_client) + if 'macaroons' in kwargs: + if not kwargs['bakery_client']: + kwargs['bakery_client'] = httpbakery.Client() + if not kwargs['bakery_client'].cookies: + kwargs['bakery_client'].cookies = GoCookieJar() + jar = kwargs['bakery_client'].cookies + for macaroon in kwargs.pop('macaroons'): + jar.set_cookie(go_to_py_cookie(macaroon)) self._connection = await Connection.connect(**kwargs) async def disconnect(self): diff --git a/modules/libjuju/juju/client/facade.py b/modules/libjuju/juju/client/facade.py index 1c7baa0..9e2aabf 100644 --- a/modules/libjuju/juju/client/facade.py +++ b/modules/libjuju/juju/client/facade.py @@ -171,13 +171,13 @@ def name_to_py(name): def strcast(kind, keep_builtins=False): - if issubclass(kind, typing.GenericMeta): - return str(kind)[1:] - if str(kind).startswith('~'): - return str(kind)[1:] if (kind in basic_types or type(kind) in basic_types) and keep_builtins is False: return kind.__name__ + if str(kind).startswith('~'): + return str(kind)[1:] + if issubclass(kind, typing.GenericMeta): + return str(kind)[1:] return kind @@ -291,6 +291,13 @@ class {}(Type): source.append("{}self.{} = {}".format(INDENT * 2, arg_name, arg_name)) + elif type(arg_type) is typing.TypeVar: + source.append("{}self.{} = {}.from_json({}) " + "if {} else None".format(INDENT * 2, + arg_name, + arg_type_name, + arg_name, + arg_name)) elif issubclass(arg_type, typing.Sequence): value_type = ( arg_type_name.__parameters__[0] @@ -326,13 +333,6 @@ class {}(Type): source.append("{}self.{} = {}".format(INDENT * 2, arg_name, arg_name)) - elif type(arg_type) is typing.TypeVar: - source.append("{}self.{} = {}.from_json({}) " - "if {} else None".format(INDENT * 2, - arg_name, - arg_type_name, - arg_name, - arg_name)) else: source.append("{}self.{} = {}".format(INDENT * 2, arg_name, @@ -434,7 +434,7 @@ def ReturnMapping(cls): return decorator -def makeFunc(cls, name, params, result, async=True): +def makeFunc(cls, name, params, result, _async=True): INDENT = " " args = Args(params) assignments = [] @@ -448,7 +448,7 @@ def makeFunc(cls, name, params, result, async=True): source = """ @ReturnMapping({rettype}) -{async}def {name}(self{argsep}{args}): +{_async}def {name}(self{argsep}{args}): ''' {docstring} Returns -> {res} @@ -460,12 +460,12 @@ def makeFunc(cls, name, params, result, async=True): version={cls.version}, params=_params) {assignments} - reply = {await}self.rpc(msg) + reply = {_await}self.rpc(msg) return reply """ - fsource = source.format(async="async " if async else "", + fsource = source.format(_async="async " if _async else "", name=name, argsep=", " if args else "", args=args, @@ -474,7 +474,7 @@ def makeFunc(cls, name, params, result, async=True): docstring=textwrap.indent(args.get_doc(), INDENT), cls=cls, assignments=assignments, - await="await " if async else "") + _await="await " if _async else "") ns = _getns() exec(fsource, ns) func = ns[name] diff --git a/modules/libjuju/juju/client/gocookies.py b/modules/libjuju/juju/client/gocookies.py index a8a0df8..3e48b8d 100644 --- a/modules/libjuju/juju/client/gocookies.py +++ b/modules/libjuju/juju/client/gocookies.py @@ -15,7 +15,7 @@ class GoCookieJar(cookiejar.FileCookieJar): to implement the actual cookie loading''' data = json.load(f) or [] now = time.time() - for cookie in map(_new_py_cookie, data): + for cookie in map(go_to_py_cookie, data): if not ignore_expires and cookie.is_expired(now): continue self.set_cookie(cookie) @@ -37,12 +37,12 @@ class GoCookieJar(cookiejar.FileCookieJar): continue if not ignore_expires and cookie.is_expired(now): continue - go_cookies.append(_new_go_cookie(cookie)) + go_cookies.append(py_to_go_cookie(cookie)) with open(filename, "w") as f: f.write(json.dumps(go_cookies)) -def _new_py_cookie(go_cookie): +def go_to_py_cookie(go_cookie): '''Convert a Go-style JSON-unmarshaled cookie into a Python cookie''' expires = None if go_cookie.get('Expires') is not None: @@ -75,7 +75,7 @@ def _new_py_cookie(go_cookie): ) -def _new_go_cookie(py_cookie): +def py_to_go_cookie(py_cookie): '''Convert a python cookie to the JSON-marshalable Go-style cookie form.''' # TODO (perhaps): # HttpOnly diff --git a/modules/libjuju/juju/client/overrides.py b/modules/libjuju/juju/client/overrides.py index 8b29de7..49ab931 100644 --- a/modules/libjuju/juju/client/overrides.py +++ b/modules/libjuju/juju/client/overrides.py @@ -15,6 +15,7 @@ __all__ = [ __patches__ = [ 'ResourcesFacade', 'AllWatcherFacade', + 'ActionFacade', ] @@ -105,6 +106,42 @@ class AllWatcherFacade(Type): return result +class ActionFacade(Type): + + class _FindTagsResults(Type): + _toSchema = {'matches': 'matches'} + _toPy = {'matches': 'matches'} + + def __init__(self, matches=None, **unknown_fields): + ''' + FindTagsResults wraps the mapping between the requested prefix and the + matching tags for each requested prefix. + + Matches map[string][]Entity `json:"matches"` + ''' + self.matches = {} + matches = matches or {} + for prefix, tags in matches.items(): + self.matches[prefix] = [_definitions.Entity.from_json(r) + for r in tags] + + @ReturnMapping(_FindTagsResults) + async def FindActionTagsByPrefix(self, prefixes): + ''' + prefixes : typing.Sequence[str] + Returns -> typing.Sequence[~Entity] + ''' + # map input types to rpc msg + _params = dict() + msg = dict(type='Action', + request='FindActionTagsByPrefix', + version=2, + params=_params) + _params['prefixes'] = prefixes + reply = await self.rpc(msg) + return reply + + class Number(_definitions.Number): """ This type represents a semver string. @@ -138,14 +175,24 @@ class Number(_definitions.Number): def __str__(self): return self.serialize() + @property + def _cmp(self): + return (self.major, self.minor, self.tag, self.patch, self.build) + def __eq__(self, other): - return ( - isinstance(other, type(self)) and - other.major == self.major and - other.minor == self.minor and - other.tag == self.tag and - other.patch == self.patch and - other.build == self.build) + return isinstance(other, type(self)) and self._cmp == other._cmp + + def __lt__(self, other): + return self._cmp < other._cmp + + def __le__(self, other): + return self._cmp <= other._cmp + + def __gt__(self, other): + return self._cmp > other._cmp + + def __ge__(self, other): + return self._cmp >= other._cmp @classmethod def from_json(cls, data): diff --git a/modules/libjuju/juju/constraints.py b/modules/libjuju/juju/constraints.py index 998862d..0050673 100644 --- a/modules/libjuju/juju/constraints.py +++ b/modules/libjuju/juju/constraints.py @@ -29,6 +29,8 @@ FACTORS = { "P": 1024 * 1024 * 1024 } +LIST_KEYS = {'tags', 'spaces'} + SNAKE1 = re.compile(r'(.)([A-Z][a-z]+)') SNAKE2 = re.compile('([a-z0-9])([A-Z])') @@ -47,8 +49,10 @@ def parse(constraints): return constraints constraints = { - normalize_key(k): normalize_value(v) for k, v in [ - s.split("=") for s in constraints.split(" ")]} + normalize_key(k): ( + normalize_list_value(v) if k in LIST_KEYS else + normalize_value(v) + ) for k, v in [s.split("=") for s in constraints.split(" ")]} return constraints @@ -72,13 +76,12 @@ def normalize_value(value): # Translate aliases to Megabytes. e.g. 1G = 10240 return int(value[:-1]) * FACTORS[value[-1:]] - if "," in value: - # Handle csv strings. - values = value.split(",") - values = [normalize_value(v) for v in values] - return values - if value.isdigit(): return int(value) return value + + +def normalize_list_value(value): + values = value.strip().split(',') + return [normalize_value(value) for value in values] diff --git a/modules/libjuju/juju/controller.py b/modules/libjuju/juju/controller.py index 957ab85..b4c544e 100644 --- a/modules/libjuju/juju/controller.py +++ b/modules/libjuju/juju/controller.py @@ -1,5 +1,7 @@ import asyncio +import json import logging +from pathlib import Path from . import errors, tag, utils from .client import client, connector @@ -48,32 +50,101 @@ class Controller: def loop(self): return self._connector.loop - async def connect(self, controller_name=None, **kwargs): + async def connect(self, *args, **kwargs): """Connect to a Juju controller. - If any arguments are specified other than controller_name, - then controller_name must be None and an explicit - connection will be made using Connection.connect - using those parameters (the 'uuid' parameter must - be absent or None). - - Otherwise, if controller_name is None, connect to the - current controller. - - Otherwise, controller_name must specify the name - of a known controller. + This supports two calling conventions: + + The controller and (optionally) authentication information can be + taken from the data files created by the Juju CLI. This convention + will be used if a ``controller_name`` is specified, or if the + ``endpoint`` is not. + + Otherwise, both the ``endpoint`` and authentication information + (``username`` and ``password``, or ``bakery_client`` and/or + ``macaroons``) are required. + + If a single positional argument is given, it will be assumed to be + the ``controller_name``. Otherwise, the first positional argument, + if any, must be the ``endpoint``. + + Available parameters are: + + :param str controller_name: Name of controller registered with the + Juju CLI. + :param str endpoint: The hostname:port of the controller to connect to. + :param str username: The username for controller-local users (or None + to use macaroon-based login.) + :param str password: The password for controller-local users. + :param str cacert: The CA certificate of the controller + (PEM formatted). + :param httpbakery.Client bakery_client: The macaroon bakery client to + to use when performing macaroon-based login. Macaroon tokens + acquired when logging will be saved to bakery_client.cookies. + If this is None, a default bakery_client will be used. + :param list macaroons: List of macaroons to load into the + ``bakery_client``. + :param asyncio.BaseEventLoop loop: The event loop to use for async + operations. + :param int max_frame_size: The maximum websocket frame size to allow. """ await self.disconnect() - if not kwargs: - await self._connector.connect_controller(controller_name) + if 'endpoint' not in kwargs and len(args) < 2: + if args and 'model_name' in kwargs: + raise TypeError('connect() got multiple values for ' + 'controller_name') + elif args: + controller_name = args[0] + else: + controller_name = kwargs.pop('controller_name', None) + await self._connector.connect_controller(controller_name, **kwargs) else: - if controller_name is not None: - raise ValueError('controller name may not be specified with other connect parameters') - if kwargs.get('uuid') is not None: - # A UUID implies a model connection, not a controller connection. - raise ValueError('model UUID specified when connecting to controller') + if 'controller_name' in kwargs: + raise TypeError('connect() got values for both ' + 'controller_name and endpoint') + if args and 'endpoint' in kwargs: + raise TypeError('connect() got multiple values for endpoint') + has_userpass = (len(args) >= 3 or + {'username', 'password'}.issubset(kwargs)) + has_macaroons = (len(args) >= 5 or not + {'bakery_client', 'macaroons'}.isdisjoint(kwargs)) + if not (has_userpass or has_macaroons): + raise TypeError('connect() missing auth params') + arg_names = [ + 'endpoint', + 'username', + 'password', + 'cacert', + 'bakery_client', + 'macaroons', + 'loop', + 'max_frame_size', + ] + for i, arg in enumerate(args): + kwargs[arg_names[i]] = arg + if 'endpoint' not in kwargs: + raise ValueError('endpoint is required ' + 'if controller_name not given') + if not ({'username', 'password'}.issubset(kwargs) or + {'bakery_client', 'macaroons'}.intersection(kwargs)): + raise ValueError('Authentication parameters are required ' + 'if controller_name not given') await self._connector.connect(**kwargs) + async def connect_current(self): + """ + .. deprecated:: 0.7.3 + Use :meth:`.connect()` instead. + """ + return await self.connect() + + async def connect_controller(self, controller_name): + """ + .. deprecated:: 0.7.3 + Use :meth:`.connect(controller_name)` instead. + """ + return await self.connect(controller_name) + async def _connect_direct(self, **kwargs): await self.disconnect() await self._connector.connect(**kwargs) @@ -127,6 +198,21 @@ class Controller: raise errors.JujuError( 'Unable to find credential: {}'.format(name)) + if credential.auth_type == 'jsonfile' and 'file' in credential.attrs: + # file creds have to be loaded before being sent to the controller + try: + # it might already be JSON + json.loads(credential.attrs['file']) + except json.JSONDecodeError: + # not valid JSON, so maybe it's a file + cred_path = Path(credential.attrs['file']) + if cred_path.exists(): + # make a copy + cred_json = credential.to_json() + credential = client.CloudCredential.from_json(cred_json) + # inline the cred + credential.attrs['file'] = cred_path.read_text() + log.debug('Uploading credential %s', name) cloud_facade = client.CloudFacade.from_connection(self.connection()) await cloud_facade.UpdateCredentials([ @@ -315,7 +401,7 @@ class Controller: Use :meth:`.list_models` instead. """ controller_facade = client.ControllerFacade.from_connection( - self.connection) + self.connection()) for attempt in (1, 2, 3): try: return await controller_facade.AllModels() diff --git a/modules/libjuju/juju/machine.py b/modules/libjuju/juju/machine.py index bd3d030..a46135c 100644 --- a/modules/libjuju/juju/machine.py +++ b/modules/libjuju/juju/machine.py @@ -14,7 +14,18 @@ log = logging.getLogger(__name__) class Machine(model.ModelEntity): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.on_change(self._workaround_1695335) + self.model.loop.create_task(self._queue_workarounds()) + + async def _queue_workarounds(self): + model = self.model + if not model.info: + await utils.run_with_interrupt(model.get_info(), + model._watch_stopping, + model.loop) + if model._watch_stopping.is_set(): + return + if model.info.agent_version < client.Number.from_json('2.2.3'): + self.on_change(self._workaround_1695335) async def _workaround_1695335(self, delta, old, new, model): """ diff --git a/modules/libjuju/juju/model.py b/modules/libjuju/juju/model.py index ac22599..37e8cd6 100644 --- a/modules/libjuju/juju/model.py +++ b/modules/libjuju/juju/model.py @@ -22,12 +22,15 @@ import yaml from . import tag, utils from .client import client, connector from .client.client import ConfigValue +from .client.client import Value from .constraints import parse as parse_constraints from .constraints import normalize_key from .delta import get_entity_class, get_entity_delta from .errors import JujuAPIError, JujuError from .exceptions import DeadEntityException from .placement import parse as parse_placement +from . import provisioner + log = logging.getLogger(__name__) @@ -410,7 +413,7 @@ class Model: `juju.client.connection.Connection.MAX_FRAME_SIZE` :param bakery_client httpbakery.Client: The bakery client to use for macaroon authorization. - :param jujudata JujuData: The source for current controller information. + :param jujudata JujuData: The source for current controller information """ self._connector = connector.Connector( loop=loop, @@ -458,42 +461,101 @@ class Model: async def __aexit__(self, exc_type, exc, tb): await self.disconnect() - async def connect(self, model_name=None, **kwargs): + async def connect(self, *args, **kwargs): """Connect to a juju model. - If any arguments are specified other than model_name, then - model_name must be None and an explicit connection will be made - using Connection.connect using those parameters (the 'uuid' - parameter must be specified). + This supports two calling conventions: - Otherwise, if model_name is None, connect to the current model. + The model and (optionally) authentication information can be taken + from the data files created by the Juju CLI. This convention will + be used if a ``model_name`` is specified, or if the ``endpoint`` + and ``uuid`` are not. - Otherwise, model_name must specify the name of a known - model. + Otherwise, all of the ``endpoint``, ``uuid``, and authentication + information (``username`` and ``password``, or ``bakery_client`` and/or + ``macaroons``) are required. - :param model_name: Format [controller:][user/]model + If a single positional argument is given, it will be assumed to be + the ``model_name``. Otherwise, the first positional argument, if any, + must be the ``endpoint``. + Available parameters are: + + :param model_name: Format [controller:][user/]model + :param str endpoint: The hostname:port of the controller to connect to. + :param str uuid: The model UUID to connect to. + :param str username: The username for controller-local users (or None + to use macaroon-based login.) + :param str password: The password for controller-local users. + :param str cacert: The CA certificate of the controller + (PEM formatted). + :param httpbakery.Client bakery_client: The macaroon bakery client to + to use when performing macaroon-based login. Macaroon tokens + acquired when logging will be saved to bakery_client.cookies. + If this is None, a default bakery_client will be used. + :param list macaroons: List of macaroons to load into the + ``bakery_client``. + :param asyncio.BaseEventLoop loop: The event loop to use for async + operations. + :param int max_frame_size: The maximum websocket frame size to allow. """ await self.disconnect() - if not kwargs: - await self._connector.connect_model(model_name) + if 'endpoint' not in kwargs and len(args) < 2: + if args and 'model_name' in kwargs: + raise TypeError('connect() got multiple values for model_name') + elif args: + model_name = args[0] + else: + model_name = kwargs.pop('model_name', None) + await self._connector.connect_model(model_name, **kwargs) else: - if kwargs.get('uuid') is None: - raise ValueError('no UUID specified when connecting to model') + if 'model_name' in kwargs: + raise TypeError('connect() got values for both ' + 'model_name and endpoint') + if args and 'endpoint' in kwargs: + raise TypeError('connect() got multiple values for endpoint') + if len(args) < 2 and 'uuid' not in kwargs: + raise TypeError('connect() missing value for uuid') + has_userpass = (len(args) >= 4 or + {'username', 'password'}.issubset(kwargs)) + has_macaroons = (len(args) >= 6 or not + {'bakery_client', 'macaroons'}.isdisjoint(kwargs)) + if not (has_userpass or has_macaroons): + raise TypeError('connect() missing auth params') + arg_names = [ + 'endpoint', + 'uuid', + 'username', + 'password', + 'cacert', + 'bakery_client', + 'macaroons', + 'loop', + 'max_frame_size', + ] + for i, arg in enumerate(args): + kwargs[arg_names[i]] = arg + if not {'endpoint', 'uuid'}.issubset(kwargs): + raise ValueError('endpoint and uuid are required ' + 'if model_name not given') + if not ({'username', 'password'}.issubset(kwargs) or + {'bakery_client', 'macaroons'}.intersection(kwargs)): + raise ValueError('Authentication parameters are required ' + 'if model_name not given') await self._connector.connect(**kwargs) await self._after_connect() async def connect_model(self, model_name): """ .. deprecated:: 0.6.2 - Use connect(model_name=model_name) instead. + Use ``connect(model_name=model_name)`` instead. """ return await self.connect(model_name=model_name) async def connect_current(self): """ .. deprecated:: 0.6.2 - Use connect instead. + Use ``connect()`` instead. """ return await self.connect() @@ -528,7 +590,7 @@ class Model: if self.is_connected(): log.debug('Closing model connection') await self._connector.disconnect() - self.info = None + self._info = None async def add_local_charm_dir(self, charm_dir, series): """Upload a local charm to the model. @@ -675,11 +737,19 @@ class Model: """ facade = client.ClientFacade.from_connection(self.connection()) - self.info = await facade.ModelInfo() + self._info = await facade.ModelInfo() log.debug('Got ModelInfo: %s', vars(self.info)) return self.info + @property + def info(self): + """Return the cached client.ModelInfo object for this Model. + + If Model.get_info() has not been called, this will return None. + """ + return self._info + def add_observer( self, callable_, entity_type=None, action=None, entity_id=None, predicate=None): @@ -880,7 +950,8 @@ class Model: (None) - starts a new machine 'lxd' - starts a new machine with one lxd container 'lxd:4' - starts a new lxd container on machine 4 - 'ssh:user@10.10.0.3' - manually provisions a machine with ssh + 'ssh:user@10.10.0.3:/path/to/private/key' - manually provision + a machine with ssh and the private key used for authentication 'zone=us-east-1a' - starts a machine in zone us-east-1s on AWS 'maas2.name' - acquire machine maas2.name on MAAS @@ -929,12 +1000,25 @@ class Model: """ params = client.AddMachineParams() - params.jobs = ['JobHostUnits'] if spec: - placement = parse_placement(spec) - if placement: - params.placement = placement[0] + if spec.startswith("ssh:"): + placement, target, private_key_path = spec.split(":") + user, host = target.split("@") + + sshProvisioner = provisioner.SSHProvisioner( + host=host, + user=user, + private_key_path=private_key_path, + ) + + params = sshProvisioner.provision_machine() + else: + placement = parse_placement(spec) + if placement: + params.placement = placement[0] + + params.jobs = ['JobHostUnits'] if constraints: params.constraints = client.Value.from_json(constraints) @@ -953,6 +1037,17 @@ class Model: if error: raise ValueError("Error adding machine: %s" % error.message) machine_id = results.machines[0].machine + + if spec: + if spec.startswith("ssh:"): + # Need to run this after AddMachines has been called, + # as we need the machine_id + await sshProvisioner.install_agent( + self.connection(), + params.nonce, + machine_id, + ) + log.debug('Added new machine %s', machine_id) return await self._wait_for_new('machine', machine_id) @@ -963,7 +1058,8 @@ class Model: :param str relation2: '[:]' """ - app_facade = client.ApplicationFacade.from_connection(self.connection()) + connection = self.connection() + app_facade = client.ApplicationFacade.from_connection(connection) log.debug( 'Adding relation %s <-> %s', relation1, relation2) @@ -1312,7 +1408,8 @@ class Model: """Destroy units by name. """ - app_facade = client.ApplicationFacade.from_connection(self.connection()) + connection = self.connection() + app_facade = client.ApplicationFacade.from_connection(connection) log.debug( 'Destroying unit%s %s', @@ -1365,11 +1462,28 @@ class Model: config[key] = ConfigValue.from_json(value) return config - def get_constraints(self): + async def get_constraints(self): """Return the machine constraints for this model. + :returns: A ``dict`` of constraints. """ - raise NotImplementedError() + constraints = {} + client_facade = client.ClientFacade.from_connection(self.connection()) + result = await client_facade.GetModelConstraints() + + # GetModelConstraints returns GetConstraintsResults which has a 'constraints' + # attribute. If no constraints have been set GetConstraintsResults.constraints + # is None. Otherwise GetConstraintsResults.constraints has an attribute for each + # possible constraint, each of these in turn will be None if they have not been + # set. + if result.constraints: + constraint_types = [a for a in dir(result.constraints) + if a in Value._toSchema.keys()] + for constraint in constraint_types: + value = getattr(result.constraints, constraint) + if value is not None: + constraints[constraint] = getattr(result.constraints, constraint) + return constraints def import_ssh_key(self, identity): """Add a public SSH key from a trusted indentity source to this model. @@ -1528,31 +1642,79 @@ class Model: config[key] = value.value await config_facade.ModelSet(config) - def set_constraints(self, constraints): + async def set_constraints(self, constraints): """Set machine constraints on this model. - :param :class:`juju.Constraints` constraints: Machine constraints - + :param dict config: Mapping of model constraints """ - raise NotImplementedError() + client_facade = client.ClientFacade.from_connection(self.connection()) + await client_facade.SetModelConstraints( + application='', + constraints=constraints) - def get_action_output(self, action_uuid, wait=-1): + async def get_action_output(self, action_uuid, wait=None): """Get the results of an action by ID. :param str action_uuid: Id of the action - :param int wait: Time in seconds to wait for action to complete - + :param int wait: Time in seconds to wait for action to complete. + :return dict: Output from action + :raises: :class:`JujuError` if invalid action_uuid """ - raise NotImplementedError() + action_facade = client.ActionFacade.from_connection( + self.connection() + ) + entity = [{'tag': tag.action(action_uuid)}] + # Cannot use self.wait_for_action as the action event has probably + # already happened and self.wait_for_action works by processing + # model deltas and checking if they match our type. If the action + # has already occured then the delta has gone. + + async def _wait_for_action_status(): + while True: + action_output = await action_facade.Actions(entity) + if action_output.results[0].status in ('completed', 'failed'): + return + else: + await asyncio.sleep(1) + await asyncio.wait_for( + _wait_for_action_status(), + timeout=wait) + action_output = await action_facade.Actions(entity) + # ActionResult.output is None if the action produced no output + if action_output.results[0].output is None: + output = {} + else: + output = action_output.results[0].output + return output - def get_action_status(self, uuid_or_prefix=None, name=None): - """Get the status of all actions, filtered by ID, ID prefix, or action name. + async def get_action_status(self, uuid_or_prefix=None, name=None): + """Get the status of all actions, filtered by ID, ID prefix, or name. :param str uuid_or_prefix: Filter by action uuid or prefix :param str name: Filter by action name """ - raise NotImplementedError() + results = {} + action_results = [] + action_facade = client.ActionFacade.from_connection( + self.connection() + ) + if name: + name_results = await action_facade.FindActionsByNames([name]) + action_results.extend(name_results.actions[0].actions) + if uuid_or_prefix: + # Collect list of actions matching uuid or prefix + matching_actions = await action_facade.FindActionTagsByPrefix( + [uuid_or_prefix]) + entities = [] + for actions in matching_actions.matches.values(): + entities = [{'tag': a.tag} for a in actions] + # Get action results matching action tags + uuid_results = await action_facade.Actions(entities) + action_results.extend(uuid_results.results) + for a in action_results: + results[tag.untag('action-', a.action.tag)] = a.status + return results def get_budget(self, budget_name): """Get budget usage info. @@ -1774,6 +1936,9 @@ class BundleHandler: self.plan = await self.client_facade.GetBundleChanges( yaml.dump(self.bundle)) + if self.plan.errors: + raise JujuError(self.plan.errors) + async def execute_plan(self): for step in self.plan.changes: method = getattr(self, step.method) @@ -1839,7 +2004,11 @@ class BundleHandler: # Fix up values, as necessary. if 'parent_id' in params: - params['parent_id'] = self.resolve(params['parent_id']) + if params['parent_id'].startswith('$addUnit'): + unit = self.resolve(params['parent_id'])[0] + params['parent_id'] = unit.machine.entity_id + else: + params['parent_id'] = self.resolve(params['parent_id']) params['constraints'] = parse_constraints( params.get('constraints')) diff --git a/modules/libjuju/juju/provisioner.py b/modules/libjuju/juju/provisioner.py new file mode 100644 index 0000000..91747a4 --- /dev/null +++ b/modules/libjuju/juju/provisioner.py @@ -0,0 +1,365 @@ +from .client import client + +import paramiko +import os +import re +import tempfile +import shlex +from subprocess import ( + CalledProcessError, +) +import uuid + + +arches = [ + [re.compile("amd64|x86_64"), "amd64"], + [re.compile("i?[3-9]86"), "i386"], + [re.compile("(arm$)|(armv.*)"), "armhf"], + [re.compile("aarch64"), "arm64"], + [re.compile("ppc64|ppc64el|ppc64le"), "ppc64el"], + [re.compile("ppc64|ppc64el|ppc64le"), "s390x"], + +] + + +def normalize_arch(rawArch): + """Normalize the architecture string.""" + for arch in arches: + if arch[0].match(rawArch): + return arch[1] + + +DETECTION_SCRIPT = """#!/bin/bash +set -e +os_id=$(grep '^ID=' /etc/os-release | tr -d '"' | cut -d= -f2) +if [ "$os_id" = 'centos' ]; then + os_version=$(grep '^VERSION_ID=' /etc/os-release | tr -d '"' | cut -d= -f2) + echo "centos$os_version" +else + lsb_release -cs +fi +uname -m +grep MemTotal /proc/meminfo +cat /proc/cpuinfo +""" + +INITIALIZE_UBUNTU_SCRIPT = """set -e +(id ubuntu &> /dev/null) || useradd -m ubuntu -s /bin/bash +umask 0077 +temp=$(mktemp) +echo 'ubuntu ALL=(ALL) NOPASSWD:ALL' > $temp +install -m 0440 $temp /etc/sudoers.d/90-juju-ubuntu +rm $temp +su ubuntu -c 'install -D -m 0600 /dev/null ~/.ssh/authorized_keys' +export authorized_keys="{}" +if [ ! -z "$authorized_keys" ]; then + su ubuntu -c 'echo $authorized_keys >> ~/.ssh/authorized_keys' +fi +""" + + +class SSHProvisioner: + """Provision a manually created machine via SSH.""" + user = "" + host = "" + private_key_path = "" + + def __init__(self, user, host, private_key_path): + self.host = host + self.user = user + self.private_key_path = private_key_path + + def _get_ssh_client(self, host, user, key): + """Return a connected Paramiko ssh object. + + :param str host: The host to connect to. + :param str user: The user to connect as. + :param str key: The private key to authenticate with. + + :return: object: A paramiko.SSHClient + :raises: :class:`paramiko.ssh_exception.SSHException` if the + connection failed + """ + + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + pkey = None + + # Read the private key into a paramiko.RSAKey + if os.path.exists(key): + with open(key, 'r') as f: + pkey = paramiko.RSAKey.from_private_key(f) + + ####################################################################### + # There is a bug in some versions of OpenSSH 4.3 (CentOS/RHEL5) where # + # the server may not send the SSH_MSG_USERAUTH_BANNER message except # + # when responding to an auth_none request. For example, paramiko will # + # attempt to use password authentication when a password is set, but # + # the server could deny that, instead requesting keyboard-interactive.# + # The hack to workaround this is to attempt a reconnect, which will # + # receive the right banner, and authentication can proceed. See the # + # following for more info: # + # https://github.com/paramiko/paramiko/issues/432 # + # https://github.com/paramiko/paramiko/pull/438 # + ####################################################################### + + try: + ssh.connect(host, port=22, username=user, pkey=pkey) + except paramiko.ssh_exception.SSHException as e: + if 'Error reading SSH protocol banner' == str(e): + # Once more, with feeling + ssh.connect(host, port=22, username=user, pkey=pkey) + else: + # Reraise the original exception + raise e + + return ssh + + def _run_command(self, ssh, cmd, pty=True): + """Run a command remotely via SSH. + + :param object ssh: The SSHClient + :param str cmd: The command to execute + :param list cmd: The `shlex.split` command to execute + :param bool pty: Whether to allocate a pty + + :return: tuple: The stdout and stderr of the command execution + :raises: :class:`CalledProcessError` if the command fails + """ + + if isinstance(cmd, str): + cmd = shlex.split(cmd) + + if type(cmd) is not list: + cmd = [cmd] + + cmds = ' '.join(cmd) + stdin, stdout, stderr = ssh.exec_command(cmds, get_pty=pty) + retcode = stdout.channel.recv_exit_status() + + if retcode > 0: + output = stderr.read().strip() + raise CalledProcessError(returncode=retcode, cmd=cmd, + output=output) + return ( + stdout.read().decode('utf-8').strip(), + stderr.read().decode('utf-8').strip() + ) + + def _init_ubuntu_user(self): + """Initialize the ubuntu user. + + :return: bool: If the initialization was successful + :raises: :class:`paramiko.ssh_exception.AuthenticationException` + if the authentication fails + """ + + # TODO: Test this on an image without the ubuntu user setup. + + auth_user = self.user + try: + # Run w/o allocating a pty, so we fail if sudo prompts for a passwd + ssh = self._get_ssh_client( + self.host, + "ubuntu", + self.private_key_path, + ) + + stdout, stderr = self._run_command(ssh, "sudo -n true", pty=False) + except paramiko.ssh_exception.AuthenticationException as e: + raise e + else: + auth_user = "ubuntu" + finally: + if ssh: + ssh.close() + + # if the above fails, run the init script as the authenticated user + + # Infer the public key + public_key = None + public_key_path = "{}.pub".format(self.private_key_path) + + if not os.path.exists(public_key_path): + raise FileNotFoundError( + "Public key '{}' doesn't exist.".format(public_key_path) + ) + + with open(public_key_path, "r") as f: + public_key = f.readline() + + script = INITIALIZE_UBUNTU_SCRIPT.format(public_key) + + try: + ssh = self._get_ssh_client( + self.host, + auth_user, + self.private_key_path, + ) + + self._run_command( + ssh, + ["sudo", "/bin/bash -c " + shlex.quote(script)], + pty=True + ) + except paramiko.ssh_exception.AuthenticationException as e: + raise e + finally: + ssh.close() + + return True + + def _detect_hardware_and_os(self, ssh): + """Detect the target hardware capabilities and OS series. + + :param object ssh: The SSHClient + :return: str: A raw string containing OS and hardware information. + """ + + info = { + 'series': '', + 'arch': '', + 'cpu-cores': '', + 'mem': '', + } + + stdout, stderr = self._run_command( + ssh, + ["sudo", "/bin/bash -c " + shlex.quote(DETECTION_SCRIPT)], + pty=True, + ) + + lines = stdout.split("\n") + info['series'] = lines[0].strip() + info['arch'] = normalize_arch(lines[1].strip()) + + memKb = re.split('\s+', lines[2])[1] + + # Convert megabytes -> kilobytes + info['mem'] = round(int(memKb) / 1024) + + # Detect available CPUs + recorded = {} + for line in lines[3:]: + physical_id = "" + print(line) + + if line.find("physical id") == 0: + physical_id = line.split(":")[1].strip() + elif line.find("cpu cores") == 0: + cores = line.split(":")[1].strip() + + if physical_id not in recorded.keys(): + info['cpu-cores'] += cores + recorded[physical_id] = True + + return info + + def provision_machine(self): + """Perform the initial provisioning of the target machine. + + :return: bool: The client.AddMachineParams + :raises: :class:`paramiko.ssh_exception.AuthenticationException` + if the upload fails + """ + params = client.AddMachineParams() + + if self._init_ubuntu_user(): + try: + + ssh = self._get_ssh_client( + self.host, + self.user, + self.private_key_path + ) + + hw = self._detect_hardware_and_os(ssh) + params.series = hw['series'] + params.instance_id = "manual:{}".format(self.host) + params.nonce = "manual:{}:{}".format( + self.host, + str(uuid.uuid4()), # a nop for Juju w/manual machines + ) + params.hardware_characteristics = { + 'arch': hw['arch'], + 'mem': int(hw['mem']), + 'cpu-cores': int(hw['cpu-cores']), + } + params.addresses = [{ + 'value': self.host, + 'type': 'ipv4', + 'scope': 'public', + }] + + except paramiko.ssh_exception.AuthenticationException as e: + raise e + finally: + ssh.close() + + return params + + async def install_agent(self, connection, nonce, machine_id): + """ + :param object connection: Connection to Juju API + :param str nonce: The nonce machine specification + :param str machine_id: The id assigned to the machine + + :return: bool: If the initialization was successful + """ + + # The path where the Juju agent should be installed. + data_dir = "/var/lib/juju" + + # Disabling this prevents `apt-get update` from running initially, so + # charms will fail to deploy + disable_package_commands = False + + client_facade = client.ClientFacade.from_connection(connection) + results = await client_facade.ProvisioningScript( + data_dir, + disable_package_commands, + machine_id, + nonce, + ) + + self._run_configure_script(results.script) + + def _run_configure_script(self, script): + """Run the script to install the Juju agent on the target machine. + + :param str script: The script returned by the ProvisioningScript API + :raises: :class:`paramiko.ssh_exception.AuthenticationException` + if the upload fails + """ + + _, tmpFile = tempfile.mkstemp() + with open(tmpFile, 'w') as f: + f.write(script) + + try: + # get ssh client + ssh = self._get_ssh_client( + self.host, + "ubuntu", + self.private_key_path, + ) + + # copy the local copy of the script to the remote machine + sftp = paramiko.SFTPClient.from_transport(ssh.get_transport()) + sftp.put( + tmpFile, + tmpFile, + ) + + # run the provisioning script + stdout, stderr = self._run_command( + ssh, + "sudo /bin/bash {}".format(tmpFile), + ) + + except paramiko.ssh_exception.AuthenticationException as e: + raise e + finally: + os.remove(tmpFile) + ssh.close() diff --git a/modules/libjuju/juju/tag.py b/modules/libjuju/juju/tag.py index 319e8f8..282e0a6 100644 --- a/modules/libjuju/juju/tag.py +++ b/modules/libjuju/juju/tag.py @@ -34,3 +34,7 @@ def user(username): def application(app_name): return _prefix('application-', app_name) + + +def action(action_uuid): + return _prefix('action-', action_uuid) diff --git a/modules/libjuju/juju/unit.py b/modules/libjuju/juju/unit.py index ce33b08..3be27f2 100644 --- a/modules/libjuju/juju/unit.py +++ b/modules/libjuju/juju/unit.py @@ -122,7 +122,7 @@ class Unit(model.ModelEntity): """Run command on this unit. :param str command: The command to run - :param int timeout: Time to wait before command is considered failed + :param int timeout: Time, in seconds, to wait before command is considered failed :returns: A :class:`juju.action.Action` instance. """ @@ -131,6 +131,10 @@ class Unit(model.ModelEntity): log.debug( 'Running `%s` on %s', command, self.name) + if timeout: + # Convert seconds to nanoseconds + timeout = int(timeout * 1000000000) + res = await action.Run( [], command, diff --git a/modules/libjuju/setup.py b/modules/libjuju/setup.py index ff8e403..67e3707 100644 --- a/modules/libjuju/setup.py +++ b/modules/libjuju/setup.py @@ -25,32 +25,36 @@ long_description = '{}\n\n{}'.format( version = here / 'VERSION' setup( - name='juju', - version=version.read_text().strip(), - packages=find_packages( - exclude=["*.tests", "*.tests.*", "tests.*", "tests"]), - install_requires=[ - 'websockets>=4.0,<5.0', - 'pyyaml>=3.0,<4.0', - 'theblues>=0.3.8,<1.0', - 'python-dateutil', - ], - include_package_data=True, - maintainer='Juju Ecosystem Engineering', - maintainer_email='juju@lists.ubuntu.com', - description=('Python library for Juju'), - long_description=long_description, - url='https://github.com/juju/python-libjuju', - license='Apache 2', - classifiers=[ - "Development Status :: 3 - Alpha", - "Intended Audience :: Developers", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", - ], - entry_points={ - 'console_scripts': [ - ], - }, - ) + name='juju', + version=version.read_text().strip(), + packages=find_packages( + exclude=["*.tests", "*.tests.*", "tests.*", "tests"]), + install_requires=[ + 'macaroonbakery>=1.1,<2.0', + 'pyRFC3339>=1.0,<2.0', + 'pyyaml>=3.0,<=4.2', + 'theblues>=0.3.8,<1.0', + 'websockets>=4.0,<7.0', + 'paramiko>=2.4.0,<3.0.0', + ], + include_package_data=True, + maintainer='Juju Ecosystem Engineering', + maintainer_email='juju@lists.ubuntu.com', + description=('Python library for Juju'), + long_description=long_description, + url='https://github.com/juju/python-libjuju', + license='Apache 2', + classifiers=[ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + ], + entry_points={ + 'console_scripts': [ + ], + }, +) diff --git a/modules/libjuju/tests/base.py b/modules/libjuju/tests/base.py index bae4b80..97eea53 100644 --- a/modules/libjuju/tests/base.py +++ b/modules/libjuju/tests/base.py @@ -1,6 +1,8 @@ import inspect import subprocess import uuid +from contextlib import contextmanager +from pathlib import Path import mock from juju.client.jujudata import FileJujuData @@ -10,10 +12,14 @@ import pytest def is_bootstrapped(): - result = subprocess.run(['juju', 'switch'], stdout=subprocess.PIPE) - return ( - result.returncode == 0 and - len(result.stdout.decode().strip()) > 0) + try: + result = subprocess.run(['juju', 'switch'], stdout=subprocess.PIPE) + return ( + result.returncode == 0 and + len(result.stdout.decode().strip()) > 0) + except FileNotFoundError: + return False + bootstrapped = pytest.mark.skipif( @@ -24,6 +30,13 @@ test_run_nonce = uuid.uuid4().hex[-4:] class CleanController(): + """ + Context manager that automatically connects and disconnects from + the currently active controller. + + Note: Unlike CleanModel, this will not create a new controller for you, + and an active controller must already be available. + """ def __init__(self): self._controller = None @@ -37,6 +50,14 @@ class CleanController(): class CleanModel(): + """ + Context manager that automatically connects to the currently active + controller, adds a fresh model, returns the connection to that model, + and automatically disconnects and cleans up the model. + + The new model is also set as the current default for the controller + connection. + """ def __init__(self, bakery_client=None): self._controller = None self._model = None @@ -77,14 +98,6 @@ class CleanModel(): return self._model - def _models(self): - result = self._orig_models() - models = result[self.controller_name]['models'] - full_model_name = '{}/{}'.format(self.user_name, self.model_name) - if full_model_name not in models: - models[full_model_name] = {'uuid': self.model_uuid} - return result - async def __aexit__(self, exc_type, exc, tb): await self._model.disconnect() await self._controller.destroy_model(self._model_uuid) @@ -115,9 +128,22 @@ class TestJujuData(FileJujuData): cmodels = all_models[self.__controller_name]['models'] cmodels[self.__model_name] = {'uuid': self.__model_uuid} return all_models ->>>>>>> New N2VC interface + updated libjuju class AsyncMock(mock.MagicMock): async def __call__(self, *args, **kwargs): return super().__call__(*args, **kwargs) + + +@contextmanager +def patch_file(filename): + """ + "Patch" a file so that its current contents are automatically restored + when the context is exited. + """ + filepath = Path(filename).expanduser() + data = filepath.read_bytes() + try: + yield + finally: + filepath.write_bytes(data) diff --git a/modules/libjuju/tests/bundle/invalid.yaml b/modules/libjuju/tests/bundle/invalid.yaml new file mode 100644 index 0000000..2e51c1e --- /dev/null +++ b/modules/libjuju/tests/bundle/invalid.yaml @@ -0,0 +1,7 @@ +applications: + myapp: + charm: cs:xenial/ubuntu-0 + num_units: 1 + to: + - 0 + - 0 diff --git a/modules/libjuju/tests/bundle/mini-bundle.yaml b/modules/libjuju/tests/bundle/mini-bundle.yaml new file mode 100644 index 0000000..e351a12 --- /dev/null +++ b/modules/libjuju/tests/bundle/mini-bundle.yaml @@ -0,0 +1,3 @@ +applications: + myapp: + charm: cs:xenial/ubuntu-0 diff --git a/modules/libjuju/tests/integration/test_application.py b/modules/libjuju/tests/integration/test_application.py index 7b780da..b705832 100644 --- a/modules/libjuju/tests/integration/test_application.py +++ b/modules/libjuju/tests/integration/test_application.py @@ -1,4 +1,5 @@ import asyncio + import pytest from .. import base @@ -11,9 +12,9 @@ MB = 1 async def test_action(event_loop): async with base.CleanModel() as model: ubuntu_app = await model.deploy( - 'mysql', + 'percona-cluster', application_name='mysql', - series='trusty', + series='xenial', channel='stable', config={ 'tuning-level': 'safest', @@ -28,11 +29,20 @@ async def test_action(event_loop): config = await ubuntu_app.get_config() assert config['tuning-level']['value'] == 'fast' + # Restore config back to default + await ubuntu_app.reset_config(['tuning-level']) + config = await ubuntu_app.get_config() + assert config['tuning-level']['value'] == 'safest' + # update and check app constraints await ubuntu_app.set_constraints({'mem': 512 * MB}) constraints = await ubuntu_app.get_constraints() assert constraints['mem'] == 512 * MB + # check action definitions + actions = await ubuntu_app.get_actions() + assert 'backup' in actions.keys() + @base.bootstrapped @pytest.mark.asyncio diff --git a/modules/libjuju/tests/integration/test_controller.py b/modules/libjuju/tests/integration/test_controller.py index 9c6f7ac..93e2883 100644 --- a/modules/libjuju/tests/integration/test_controller.py +++ b/modules/libjuju/tests/integration/test_controller.py @@ -1,8 +1,10 @@ import asyncio -import pytest +import subprocess import uuid from juju.client.connection import Connection +from juju.client.jujudata import FileJujuData +from juju.controller import Controller from juju.errors import JujuAPIError import pytest @@ -168,3 +170,31 @@ async def test_add_destroy_model_by_uuid(event_loop): await asyncio.wait_for(_wait_for_model_gone(controller, model_name), timeout=60) + + +# this test must be run serially because it modifies the login password +@pytest.mark.serial +@base.bootstrapped +@pytest.mark.asyncio +async def test_macaroon_auth(event_loop): + jujudata = FileJujuData() + account = jujudata.accounts()[jujudata.current_controller()] + with base.patch_file('~/.local/share/juju/accounts.yaml'): + if 'password' in account: + # force macaroon auth by "changing" password to current password + result = subprocess.run( + ['juju', 'change-user-password'], + input='{0}\n{0}\n'.format(account['password']), + universal_newlines=True, + stderr=subprocess.PIPE) + assert result.returncode == 0, ('Failed to change password: ' + '{}'.format(result.stderr)) + controller = Controller() + try: + await controller.connect() + assert controller.is_connected() + finally: + if controller.is_connected(): + await controller.disconnect() + async with base.CleanModel(): + pass # create and login to model works diff --git a/modules/libjuju/tests/integration/test_macaroon_auth.py b/modules/libjuju/tests/integration/test_macaroon_auth.py new file mode 100644 index 0000000..9911c41 --- /dev/null +++ b/modules/libjuju/tests/integration/test_macaroon_auth.py @@ -0,0 +1,108 @@ +import logging +import os + +import macaroonbakery.bakery as bakery +import macaroonbakery.httpbakery as httpbakery +import macaroonbakery.httpbakery.agent as agent +from juju.errors import JujuAPIError +from juju.model import Model + +import pytest + +from .. import base + +log = logging.getLogger(__name__) + + +@base.bootstrapped +@pytest.mark.asyncio +@pytest.mark.xfail +async def test_macaroon_auth(event_loop): + auth_info, username = agent_auth_info() + # Create a bakery client that can do agent authentication. + client = httpbakery.Client( + key=auth_info.key, + interaction_methods=[agent.AgentInteractor(auth_info)], + ) + + async with base.CleanModel(bakery_client=client) as m: + async with await m.get_controller() as c: + await c.grant_model(username, m.info.uuid, 'admin') + async with Model( + jujudata=NoAccountsJujuData(m._connector.jujudata), + bakery_client=client, + ): + pass + + +@base.bootstrapped +@pytest.mark.asyncio +@pytest.mark.xfail +async def test_macaroon_auth_with_bad_key(event_loop): + auth_info, username = agent_auth_info() + # Use a random key rather than the correct key. + auth_info = auth_info._replace(key=bakery.generate_key()) + # Create a bakery client can do agent authentication. + client = httpbakery.Client( + key=auth_info.key, + interaction_methods=[agent.AgentInteractor(auth_info)], + ) + + async with base.CleanModel(bakery_client=client) as m: + async with await m.get_controller() as c: + await c.grant_model(username, m.info.uuid, 'admin') + try: + async with Model( + jujudata=NoAccountsJujuData(m._connector.jujudata), + bakery_client=client, + ): + pytest.fail('Should not be able to connect with invalid key') + except httpbakery.BakeryException: + # We're expecting this because we're using the + # wrong key. + pass + + +@base.bootstrapped +@pytest.mark.asyncio +async def test_macaroon_auth_with_unauthorized_user(event_loop): + auth_info, username = agent_auth_info() + # Create a bakery client can do agent authentication. + client = httpbakery.Client( + key=auth_info.key, + interaction_methods=[agent.AgentInteractor(auth_info)], + ) + async with base.CleanModel(bakery_client=client) as m: + # Note: no grant of rights to the agent user. + try: + async with Model( + jujudata=NoAccountsJujuData(m._connector.jujudata), + bakery_client=client, + ): + pytest.fail('Should not be able to connect without grant') + except (JujuAPIError, httpbakery.DischargeError): + # We're expecting this because we're using the + # wrong user name. + pass + + +def agent_auth_info(): + agent_data = os.environ.get('TEST_AGENTS') + if agent_data is None: + pytest.skip('skipping macaroon_auth because no TEST_AGENTS ' + 'environment variable is set') + auth_info = agent.read_auth_info(agent_data) + if len(auth_info.agents) != 1: + raise Exception('TEST_AGENTS agent data requires exactly one agent') + return auth_info, auth_info.agents[0].username + + +class NoAccountsJujuData: + def __init__(self, jujudata): + self.__jujudata = jujudata + + def __getattr__(self, name): + return getattr(self.__jujudata, name) + + def accounts(self): + return {} diff --git a/modules/libjuju/tests/integration/test_machine.py b/modules/libjuju/tests/integration/test_machine.py index 8957ae1..9a5f075 100644 --- a/modules/libjuju/tests/integration/test_machine.py +++ b/modules/libjuju/tests/integration/test_machine.py @@ -26,18 +26,15 @@ async def test_status(event_loop): assert machine.agent_status == 'pending' assert not machine.agent_version + # there is some inconsistency in the capitalization of status_message + # between different providers await asyncio.wait_for( - model.block_until(lambda: (machine.status == 'running' and - machine.agent_status == 'started' and - machine.agent_version is not None)), + model.block_until( + lambda: (machine.status == 'running' and + machine.status_message.lower() == 'running' and + machine.agent_status == 'started')), timeout=480) - assert machine.status == 'running' - # there is some inconsistency in the message case between providers - assert machine.status_message.lower() == 'running' - assert machine.agent_status == 'started' - assert machine.agent_version.major >= 2 - @base.bootstrapped @pytest.mark.asyncio diff --git a/modules/libjuju/tests/integration/test_model.py b/modules/libjuju/tests/integration/test_model.py index ba2da92..1cba79a 100644 --- a/modules/libjuju/tests/integration/test_model.py +++ b/modules/libjuju/tests/integration/test_model.py @@ -6,6 +6,12 @@ from pathlib import Path from juju.client.client import ConfigValue, ApplicationFacade from juju.model import Model, ModelObserver from juju.utils import block_until, run_with_interrupt +from juju.errors import JujuError + +import os +import pylxd +import time +import uuid import pytest @@ -20,7 +26,6 @@ SSH_KEY = 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCsYMJGNGG74HAJha3n2CFmWYsOOaORn @base.bootstrapped @pytest.mark.asyncio async def test_deploy_local_bundle(event_loop): - from pathlib import Path tests_dir = Path(__file__).absolute().parent.parent bundle_path = tests_dir / 'bundle' mini_bundle_file_path = bundle_path / 'mini-bundle.yaml' @@ -33,6 +38,16 @@ async def test_deploy_local_bundle(event_loop): assert app in model.applications +@base.bootstrapped +@pytest.mark.asyncio +async def test_deploy_invalid_bundle(event_loop): + tests_dir = Path(__file__).absolute().parent.parent + bundle_path = tests_dir / 'bundle' / 'invalid.yaml' + async with base.CleanModel() as model: + with pytest.raises(JujuError): + await model.deploy(str(bundle_path)) + + @base.bootstrapped @pytest.mark.asyncio async def test_deploy_local_charm(event_loop): @@ -110,6 +125,114 @@ async def test_add_machine(event_loop): assert len(model.machines) == 0 +@base.bootstrapped +@pytest.mark.asyncio +async def test_add_manual_machine_ssh(event_loop): + + # Verify controller is localhost + async with base.CleanController() as controller: + cloud = await controller.get_cloud() + if cloud != "localhost": + pytest.skip('Skipping because test requires lxd.') + + async with base.CleanModel() as model: + private_key_path = os.path.expanduser( + "~/.local/share/juju/ssh/juju_id_rsa" + ) + public_key_path = os.path.expanduser( + "~/.local/share/juju/ssh/juju_id_rsa.pub" + ) + + # Use the self-signed cert generated by lxc on first run + crt = os.path.expanduser('~/snap/lxd/current/.config/lxc/client.crt') + assert os.path.exists(crt) + + key = os.path.expanduser('~/snap/lxd/current/.config/lxc/client.key') + assert os.path.exists(key) + + client = pylxd.Client( + endpoint="https://127.0.0.1:8443", + cert=(crt, key), + verify=False, + ) + + test_name = "test-{}-add-manual-machine-ssh".format( + uuid.uuid4().hex[-4:] + ) + + # create profile w/cloud-init and juju ssh key + public_key = "" + with open(public_key_path, "r") as f: + public_key = f.readline() + + profile = client.profiles.create( + test_name, + config={'user.user-data': '#cloud-config\nssh_authorized_keys:\n- {}'.format(public_key)}, + devices={ + 'root': {'path': '/', 'pool': 'default', 'type': 'disk'}, + 'eth0': { + 'nictype': 'bridged', + 'parent': 'lxdbr0', + 'type': 'nic' + } + } + ) + + # create lxc machine + config = { + 'name': test_name, + 'source': { + 'type': 'image', + 'alias': 'xenial', + 'mode': 'pull', + 'protocol': 'simplestreams', + 'server': 'https://cloud-images.ubuntu.com/releases', + }, + 'profiles': [test_name], + } + container = client.containers.create(config, wait=True) + container.start(wait=True) + + def wait_for_network(container, timeout=30): + """Wait for eth0 to have an ipv4 address.""" + starttime = time.time() + while(time.time() < starttime + timeout): + time.sleep(1) + if 'eth0' in container.state().network: + addresses = container.state().network['eth0']['addresses'] + if len(addresses) > 0: + if addresses[0]['family'] == 'inet': + return addresses[0] + return None + + host = wait_for_network(container) + + # HACK: We need to give sshd a chance to bind to the interface, + # and pylxd's container.execute seems to be broken and fails and/or + # hangs trying to properly check if the service is up. + time.sleep(5) + + if host: + # add a new manual machine + machine1 = await model.add_machine(spec='ssh:{}@{}:{}'.format( + "ubuntu", + host['address'], + private_key_path, + )) + + assert len(model.machines) == 1 + + res = await machine1.destroy(force=True) + + assert res is None + assert len(model.machines) == 0 + + container.stop(wait=True) + container.delete(wait=True) + + profile.delete() + + @base.bootstrapped @pytest.mark.asyncio async def test_relate(event_loop): @@ -269,6 +392,14 @@ async def test_config(event_loop): assert result['extra-info'].source == 'model' assert result['extra-info'].value == 'booyah' +@base.bootstrapped +@pytest.mark.asyncio +async def test_set_constraints(event_loop): + async with base.CleanModel() as model: + await model.set_constraints({'cpu-power': 1}) + cons = await model.get_constraints() + assert cons['cpu_power'] == 1 + # @base.bootstrapped # @pytest.mark.asyncio # async def test_grant(event_loop) diff --git a/modules/libjuju/tests/integration/test_unit.py b/modules/libjuju/tests/integration/test_unit.py index 8b2251c..bb34969 100644 --- a/modules/libjuju/tests/integration/test_unit.py +++ b/modules/libjuju/tests/integration/test_unit.py @@ -25,6 +25,18 @@ async def test_run(event_loop): assert 'Stdout' in action.results break + for unit in app.units: + action = await unit.run('sleep 1', timeout=0.5) + assert isinstance(action, Action) + assert action.status == 'failed' + break + + for unit in app.units: + action = await unit.run('sleep 0.5', timeout=2) + assert isinstance(action, Action) + assert action.status == 'completed' + break + @base.bootstrapped @pytest.mark.asyncio @@ -46,6 +58,10 @@ async def test_run_action(event_loop): for unit in app.units: action = await run_action(unit) assert action.results == {'dir': '/var/git/myrepo.git'} + out = await model.get_action_output(action.entity_id, wait=5) + assert out == {'dir': '/var/git/myrepo.git'} + status = await model.get_action_status(uuid_or_prefix=action.entity_id) + assert status[action.entity_id] == 'completed' break diff --git a/modules/libjuju/tests/unit/test_client.py b/modules/libjuju/tests/unit/test_client.py index 42134df..1d18bf9 100644 --- a/modules/libjuju/tests/unit/test_client.py +++ b/modules/libjuju/tests/unit/test_client.py @@ -4,7 +4,6 @@ Tests for generated client code """ import mock - from juju.client import client diff --git a/modules/libjuju/tests/unit/test_constraints.py b/modules/libjuju/tests/unit/test_constraints.py index 00b9156..3c52090 100644 --- a/modules/libjuju/tests/unit/test_constraints.py +++ b/modules/libjuju/tests/unit/test_constraints.py @@ -32,6 +32,12 @@ class TestConstraints(unittest.TestCase): self.assertEqual(_("10G"), 10 * 1024) self.assertEqual(_("10M"), 10) self.assertEqual(_("10"), 10) + self.assertEqual(_("foo,bar"), "foo,bar") + + def test_normalize_list_val(self): + _ = constraints.normalize_list_value + + self.assertEqual(_("foo"), ["foo"]) self.assertEqual(_("foo,bar"), ["foo", "bar"]) def test_parse_constraints(self): @@ -43,6 +49,9 @@ class TestConstraints(unittest.TestCase): ) self.assertEqual( - _("mem=10G foo=bar,baz"), - {"mem": 10 * 1024, "foo": ["bar", "baz"]} + _("mem=10G foo=bar,baz tags=tag1 spaces=space1,space2"), + {"mem": 10 * 1024, + "foo": "bar,baz", + "tags": ["tag1"], + "spaces": ["space1", "space2"]} ) diff --git a/modules/libjuju/tests/unit/test_controller.py b/modules/libjuju/tests/unit/test_controller.py new file mode 100644 index 0000000..44f488f --- /dev/null +++ b/modules/libjuju/tests/unit/test_controller.py @@ -0,0 +1,110 @@ +import asynctest +import mock +from pathlib import Path +from tempfile import NamedTemporaryFile + +from juju.controller import Controller +from juju.client import client + +from .. import base + + +class TestControllerConnect(asynctest.TestCase): + @asynctest.patch('juju.client.connector.Connector.connect_controller') + async def test_no_args(self, mock_connect_controller): + c = Controller() + await c.connect() + mock_connect_controller.assert_called_once_with(None) + + @asynctest.patch('juju.client.connector.Connector.connect_controller') + async def test_with_controller_name(self, mock_connect_controller): + c = Controller() + await c.connect(controller_name='foo') + mock_connect_controller.assert_called_once_with('foo') + + @asynctest.patch('juju.client.connector.Connector.connect') + async def test_with_endpoint_and_no_auth(self, mock_connect): + c = Controller() + with self.assertRaises(TypeError): + await c.connect(endpoint='0.1.2.3:4566') + self.assertEqual(mock_connect.call_count, 0) + + @asynctest.patch('juju.client.connector.Connector.connect') + async def test_with_endpoint_and_userpass(self, mock_connect): + c = Controller() + with self.assertRaises(TypeError): + await c.connect(endpoint='0.1.2.3:4566', username='dummy') + await c.connect(endpoint='0.1.2.3:4566', + username='user', + password='pass') + mock_connect.assert_called_once_with(endpoint='0.1.2.3:4566', + username='user', + password='pass') + + @asynctest.patch('juju.client.connector.Connector.connect') + async def test_with_endpoint_and_bakery_client(self, mock_connect): + c = Controller() + await c.connect(endpoint='0.1.2.3:4566', bakery_client='bakery') + mock_connect.assert_called_once_with(endpoint='0.1.2.3:4566', + bakery_client='bakery') + + @asynctest.patch('juju.client.connector.Connector.connect') + async def test_with_endpoint_and_macaroons(self, mock_connect): + c = Controller() + await c.connect(endpoint='0.1.2.3:4566', + macaroons=['macaroon']) + mock_connect.assert_called_with(endpoint='0.1.2.3:4566', + macaroons=['macaroon']) + await c.connect(endpoint='0.1.2.3:4566', + bakery_client='bakery', + macaroons=['macaroon']) + mock_connect.assert_called_with(endpoint='0.1.2.3:4566', + bakery_client='bakery', + macaroons=['macaroon']) + + @asynctest.patch('juju.client.connector.Connector.connect_controller') + @asynctest.patch('juju.client.connector.Connector.connect') + async def test_with_posargs(self, mock_connect, mock_connect_controller): + c = Controller() + await c.connect('foo') + mock_connect_controller.assert_called_once_with('foo') + with self.assertRaises(TypeError): + await c.connect('endpoint', 'user') + await c.connect('endpoint', 'user', 'pass') + mock_connect.assert_called_once_with(endpoint='endpoint', + username='user', + password='pass') + await c.connect('endpoint', 'user', 'pass', 'cacert', 'bakery', + 'macaroons', 'loop', 'max_frame_size') + mock_connect.assert_called_with(endpoint='endpoint', + username='user', + password='pass', + cacert='cacert', + bakery_client='bakery', + macaroons='macaroons', + loop='loop', + max_frame_size='max_frame_size') + + @asynctest.patch('juju.client.client.CloudFacade') + async def test_file_cred(self, mock_cf): + with NamedTemporaryFile() as tempfile: + tempfile.close() + temppath = Path(tempfile.name) + temppath.write_text('cred-test') + cred = client.CloudCredential(auth_type='jsonfile', + attrs={'file': tempfile.name}) + jujudata = mock.MagicMock() + c = Controller(jujudata=jujudata) + c._connector = base.AsyncMock() + up_creds = base.AsyncMock() + mock_cf.from_connection().UpdateCredentials = up_creds + await c.add_credential( + name='name', + credential=cred, + cloud='cloud', + owner='owner', + ) + assert up_creds.called + new_cred = up_creds.call_args[0][0][0].credential + assert cred.attrs['file'] == tempfile.name + assert new_cred.attrs['file'] == 'cred-test' diff --git a/modules/libjuju/tests/unit/test_gocookies.py b/modules/libjuju/tests/unit/test_gocookies.py new file mode 100644 index 0000000..033a0e9 --- /dev/null +++ b/modules/libjuju/tests/unit/test_gocookies.py @@ -0,0 +1,244 @@ +""" +Tests for the gocookies code. +""" +import os +import shutil +import tempfile +import unittest +import urllib.request + +import pyrfc3339 +from juju.client.gocookies import GoCookieJar + +# cookie_content holds the JSON contents of a Go-produced +# cookie file (reformatted so it's not all on one line but +# otherwise unchanged). +cookie_content = """ +[ + { + "CanonicalHost": "bar.com", + "Creation": "2017-11-17T08:53:55.088820092Z", + "Domain": "bar.com", + "Expires": "2345-11-15T18:16:08Z", + "HostOnly": true, + "HttpOnly": false, + "LastAccess": "2017-11-17T08:53:55.088822562Z", + "Name": "bar", + "Path": "/", + "Persistent": true, + "Secure": false, + "Updated": "2017-11-17T08:53:55.088822562Z", + "Value": "bar-value" + }, + { + "CanonicalHost": "x.foo.com", + "Creation": "2017-11-17T08:53:55.088814857Z", + "Domain": "x.foo.com", + "Expires": "2345-11-15T18:16:05Z", + "HostOnly": true, + "HttpOnly": false, + "LastAccess": "2017-11-17T08:53:55.088884015Z", + "Name": "foo", + "Path": "/path", + "Persistent": true, + "Secure": false, + "Updated": "2017-11-17T08:53:55.088814857Z", + "Value": "foo-path-value" + }, + { + "CanonicalHost": "x.foo.com", + "Creation": "2017-11-17T08:53:55.088814857Z", + "Domain": "foo.com", + "Expires": "2345-11-15T18:16:06Z", + "HostOnly": false, + "HttpOnly": false, + "LastAccess": "2017-11-17T08:53:55.088919437Z", + "Name": "foo4", + "Path": "/path", + "Persistent": true, + "Secure": false, + "Updated": "2017-11-17T08:53:55.088814857Z", + "Value": "foo4-value" + }, + { + "CanonicalHost": "x.foo.com", + "Creation": "2017-11-17T08:53:55.088790709Z", + "Domain": "x.foo.com", + "Expires": "2345-11-15T18:16:01Z", + "HostOnly": true, + "HttpOnly": false, + "LastAccess": "2017-11-17T08:53:55.088884015Z", + "Name": "foo", + "Path": "/", + "Persistent": true, + "Secure": false, + "Updated": "2017-11-17T08:53:55.088790709Z", + "Value": "foo-value" + }, + { + "CanonicalHost": "x.foo.com", + "Creation": "2017-11-17T08:53:55.088790709Z", + "Domain": "foo.com", + "Expires": "2345-11-15T18:16:02Z", + "HostOnly": false, + "HttpOnly": false, + "LastAccess": "2017-11-17T08:53:55.088919437Z", + "Name": "foo1", + "Path": "/", + "Persistent": true, + "Secure": false, + "Updated": "2017-11-17T08:53:55.088790709Z", + "Value": "foo1-value" + }, + { + "CanonicalHost": "x.foo.com", + "Creation": "2017-11-17T08:53:55.088790709Z", + "Domain": "x.foo.com", + "Expires": "2345-11-15T18:16:03Z", + "HostOnly": true, + "HttpOnly": false, + "LastAccess": "2017-11-17T08:53:55.088850252Z", + "Name": "foo2", + "Path": "/", + "Persistent": true, + "Secure": true, + "Updated": "2017-11-17T08:53:55.088790709Z", + "Value": "foo2-value" + }, + { + "CanonicalHost": "x.foo.com", + "Creation": "2017-11-17T08:53:55.088790709Z", + "Domain": "foo.com", + "Expires": "2345-11-15T18:16:04Z", + "HostOnly": false, + "HttpOnly": false, + "LastAccess": "2017-11-17T08:53:55.088919437Z", + "Name": "foo3", + "Path": "/", + "Persistent": true, + "Secure": false, + "Updated": "2017-11-17T08:53:55.088790709Z", + "Value": "foo3-value" + } +] +""" + +# cookie_content_queries holds a set of queries +# that were automatically generated by running +# the queries on the above cookie_content data +# and printing the results. +cookie_content_queries = [ + ('http://x.foo.com', [ + ('foo', 'foo-value'), + ('foo1', 'foo1-value'), + ('foo3', 'foo3-value'), + ]), + ('https://x.foo.com', [ + ('foo', 'foo-value'), + ('foo1', 'foo1-value'), + ('foo2', 'foo2-value'), + ('foo3', 'foo3-value'), + ]), + ('http://arble.foo.com', [ + ('foo1', 'foo1-value'), + ('foo3', 'foo3-value'), + ]), + ('http://arble.com', [ + ]), + ('http://x.foo.com/path/x', [ + ('foo', 'foo-path-value'), + ('foo4', 'foo4-value'), + ('foo', 'foo-value'), + ('foo1', 'foo1-value'), + ('foo3', 'foo3-value'), + ]), + ('http://arble.foo.com/path/x', [ + ('foo4', 'foo4-value'), + ('foo1', 'foo1-value'), + ('foo3', 'foo3-value'), + ]), + ('http://foo.com/path/x', [ + ('foo4', 'foo4-value'), + ('foo1', 'foo1-value'), + ('foo3', 'foo3-value'), + ]), +] + + +class TestGoCookieJar(unittest.TestCase): + def setUp(self): + self.dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.dir) + + def test_readcookies(self): + jar = self.load_jar(cookie_content) + self.assert_jar_queries(jar, cookie_content_queries) + + def test_roundtrip(self): + jar = self.load_jar(cookie_content) + filename2 = os.path.join(self.dir, 'cookies2') + jar.save(filename=filename2) + jar = GoCookieJar() + jar.load(filename=filename2) + self.assert_jar_queries(jar, cookie_content_queries) + + def test_expiry_time(self): + content = '''[ + { + "CanonicalHost": "bar.com", + "Creation": "2017-11-17T08:53:55.088820092Z", + "Domain": "bar.com", + "Expires": "2345-11-15T18:16:08Z", + "HostOnly": true, + "HttpOnly": false, + "LastAccess": "2017-11-17T08:53:55.088822562Z", + "Name": "bar", + "Path": "/", + "Persistent": true, + "Secure": false, + "Updated": "2017-11-17T08:53:55.088822562Z", + "Value": "bar-value" + } + ]''' + jar = self.load_jar(content) + got_expires = tuple(jar)[0].expires + want_expires = int(pyrfc3339.parse('2345-11-15T18:16:08Z').timestamp()) + self.assertEqual(got_expires, want_expires) + + def load_jar(self, content): + filename = os.path.join(self.dir, 'cookies') + with open(filename, 'x') as f: + f.write(content) + jar = GoCookieJar() + jar.load(filename=filename) + return jar + + def assert_jar_queries(self, jar, queries): + '''Assert that all the given queries (see cookie_content_queries) + are satisfied when run on the given cookie jar. + :param jar CookieJar: the cookie jar to query + :param queries: the queries to run. + ''' + for url, want_cookies in queries: + req = urllib.request.Request(url) + jar.add_cookie_header(req) + # We can't use SimpleCookie to find out what cookies + # have been presented, because SimpleCookie + # only allows one cookie with a given name, + # so we naively parse the cookies ourselves, which + # is OK because we know we don't have to deal + # with any complex cases. + + cookie_header = req.get_header('Cookie') + got_cookies = [] + if cookie_header is not None: + got_cookies = [ + tuple(part.split('=')) + for part in cookie_header.split('; ') + ] + got_cookies.sort() + want_cookies = list(want_cookies) + want_cookies.sort() + self.assertEqual(got_cookies, want_cookies, msg='query {}; got {}; want {}'.format(url, got_cookies, want_cookies)) diff --git a/modules/libjuju/tests/unit/test_model.py b/modules/libjuju/tests/unit/test_model.py index 2e33236..2753d85 100644 --- a/modules/libjuju/tests/unit/test_model.py +++ b/modules/libjuju/tests/unit/test_model.py @@ -5,6 +5,7 @@ import mock import asynctest from juju.client.jujudata import FileJujuData +from juju.model import Model def _make_delta(entity, type_, data=None): @@ -159,44 +160,105 @@ class TestContextManager(asynctest.TestCase): pass +@asynctest.patch('juju.model.Model._after_connect') class TestModelConnect(asynctest.TestCase): @asynctest.patch('juju.client.connector.Connector.connect_model') - @asynctest.patch('juju.model.Model._after_connect') - async def test_model_connect_no_args(self, mock_after_connect, mock_connect_model): - from juju.model import Model + async def test_no_args(self, mock_connect_model, _): m = Model() await m.connect() mock_connect_model.assert_called_once_with(None) @asynctest.patch('juju.client.connector.Connector.connect_model') - @asynctest.patch('juju.model.Model._after_connect') - async def test_model_connect_with_model_name(self, mock_after_connect, mock_connect_model): - from juju.model import Model + async def test_with_model_name(self, mock_connect_model, _): m = Model() await m.connect(model_name='foo') mock_connect_model.assert_called_once_with('foo') @asynctest.patch('juju.client.connector.Connector.connect_model') - @asynctest.patch('juju.model.Model._after_connect') - async def test_model_connect_with_endpoint_but_no_uuid( - self, - mock_after_connect, - mock_connect_model, - ): - from juju.model import Model + async def test_with_endpoint_but_no_uuid(self, mock_connect_model, _): m = Model() - with self.assertRaises(ValueError): + with self.assertRaises(TypeError): await m.connect(endpoint='0.1.2.3:4566') self.assertEqual(mock_connect_model.call_count, 0) @asynctest.patch('juju.client.connector.Connector.connect') - @asynctest.patch('juju.model.Model._after_connect') - async def test_model_connect_with_endpoint_and_uuid( - self, - mock_after_connect, - mock_connect, - ): - from juju.model import Model + async def test_with_endpoint_and_uuid_no_auth(self, mock_connect, _): + m = Model() + with self.assertRaises(TypeError): + await m.connect(endpoint='0.1.2.3:4566', uuid='some-uuid') + self.assertEqual(mock_connect.call_count, 0) + + @asynctest.patch('juju.client.connector.Connector.connect') + async def test_with_endpoint_and_uuid_with_userpass(self, mock_connect, _): + m = Model() + with self.assertRaises(TypeError): + await m.connect(endpoint='0.1.2.3:4566', + uuid='some-uuid', + username='user') + await m.connect(endpoint='0.1.2.3:4566', + uuid='some-uuid', + username='user', + password='pass') + mock_connect.assert_called_once_with(endpoint='0.1.2.3:4566', + uuid='some-uuid', + username='user', + password='pass') + + @asynctest.patch('juju.client.connector.Connector.connect') + async def test_with_endpoint_and_uuid_with_bakery(self, mock_connect, _): m = Model() - await m.connect(endpoint='0.1.2.3:4566', uuid='some-uuid') - mock_connect.assert_called_once_with(endpoint='0.1.2.3:4566', uuid='some-uuid') + await m.connect(endpoint='0.1.2.3:4566', + uuid='some-uuid', + bakery_client='bakery') + mock_connect.assert_called_once_with(endpoint='0.1.2.3:4566', + uuid='some-uuid', + bakery_client='bakery') + + @asynctest.patch('juju.client.connector.Connector.connect') + async def test_with_endpoint_and_uuid_with_macaroon(self, mock_connect, _): + m = Model() + with self.assertRaises(TypeError): + await m.connect(endpoint='0.1.2.3:4566', + uuid='some-uuid', + username='user') + await m.connect(endpoint='0.1.2.3:4566', + uuid='some-uuid', + macaroons=['macaroon']) + mock_connect.assert_called_with(endpoint='0.1.2.3:4566', + uuid='some-uuid', + macaroons=['macaroon']) + await m.connect(endpoint='0.1.2.3:4566', + uuid='some-uuid', + bakery_client='bakery', + macaroons=['macaroon']) + mock_connect.assert_called_with(endpoint='0.1.2.3:4566', + uuid='some-uuid', + bakery_client='bakery', + macaroons=['macaroon']) + + @asynctest.patch('juju.client.connector.Connector.connect_model') + @asynctest.patch('juju.client.connector.Connector.connect') + async def test_with_posargs(self, mock_connect, mock_connect_model, _): + m = Model() + await m.connect('foo') + mock_connect_model.assert_called_once_with('foo') + with self.assertRaises(TypeError): + await m.connect('endpoint', 'uuid') + with self.assertRaises(TypeError): + await m.connect('endpoint', 'uuid', 'user') + await m.connect('endpoint', 'uuid', 'user', 'pass') + mock_connect.assert_called_once_with(endpoint='endpoint', + uuid='uuid', + username='user', + password='pass') + await m.connect('endpoint', 'uuid', 'user', 'pass', 'cacert', 'bakery', + 'macaroons', 'loop', 'max_frame_size') + mock_connect.assert_called_with(endpoint='endpoint', + uuid='uuid', + username='user', + password='pass', + cacert='cacert', + bakery_client='bakery', + macaroons='macaroons', + loop='loop', + max_frame_size='max_frame_size') diff --git a/modules/libjuju/tox.ini b/modules/libjuju/tox.ini index ce421d6..e0d6a31 100644 --- a/modules/libjuju/tox.ini +++ b/modules/libjuju/tox.ini @@ -4,16 +4,21 @@ # and then run "tox" from this directory. [tox] -envlist = lint,py35 +envlist = lint,py3 skipsdist=True +[pytest] +markers = + serial: mark a test that must run by itself + [testenv] basepython=python3 usedevelop=True # for testing with other python versions -commands = py.test -ra -v -s -x -n auto {posargs} +commands = py.test --tb native -ra -v -s -n auto -k 'not integration' -m 'not serial' {posargs} passenv = HOME + TEST_AGENTS deps = pytest pytest-asyncio @@ -22,9 +27,20 @@ deps = asynctest ipdb -[testenv:py35] -# default tox env excludes integration tests -commands = py.test -ra -v -s -x -n auto -k 'not integration' {posargs} +[testenv:py3] +# default tox env excludes integration and serial tests +commands = + # These need to be installed in a specific order + pip install urllib3==1.22 + pip install pylxd + py.test --tb native -ra -v -s -n auto -k 'not integration' -m 'not serial' {posargs} + +[testenv:lint] +envdir = {toxworkdir}/py3 +commands = + flake8 --ignore E501 {posargs} juju tests +deps = + flake8 [testenv:lint] envdir = {toxworkdir}/py35 @@ -34,7 +50,17 @@ deps = flake8 [testenv:integration] -envdir = {toxworkdir}/py35 +envdir = {toxworkdir}/py3 +commands = py.test --tb native -ra -v -s -n auto -k 'integration' -m 'not serial' {posargs} + +[testenv:serial] +# tests that can't be run in parallel +envdir = {toxworkdir}/py3 +commands = py.test --tb native -ra -v -s {posargs:-m 'serial'} + +[testenv:example] +envdir = {toxworkdir}/py3 +commands = python {posargs} [flake8] exclude = juju/client/_* diff --git a/n2vc/vnf.py b/n2vc/vnf.py index 1b9efa8..7c39fa1 100644 --- a/n2vc/vnf.py +++ b/n2vc/vnf.py @@ -402,6 +402,29 @@ class N2VC: ) raise + async def GetPrimitiveStatus(self, model_name, uuid): + results = None + try: + if not self.authenticated: + await self.login() + + # FIXME: This is hard-coded until model-per-ns is added + model_name = 'default' + + model = await self.controller.get_model(model_name) + + results = await model.get_action_output(uuid) + + await model.disconnect() + except Exception as e: + self.log.debug( + "Caught exception while getting primitive status: {}".format(e) + ) + raise N2VCPrimitiveExecutionFailed(e) + + return results + + async def ExecutePrimitive(self, model_name, application_name, primitive, callback, *callback_args, **params): """Execute a primitive of a charm for Day 1 or Day 2 configuration. @@ -432,21 +455,27 @@ class N2VC: if primitive == 'config': # config is special, and expecting params to be a dictionary - self.log.debug("Setting charm configuration for {}".format(application_name)) - self.log.debug(params['params']) - await self.set_config(model, application_name, params['params']) + await self.set_config( + model, + application_name, + params['params'], + ) else: app = await self.get_application(model, application_name) if app: # Run against the first (and probably only) unit in the app unit = app.units[0] if unit: - self.log.debug("Executing primitive {}".format(primitive)) + self.log.debug( + "Executing primitive {}".format(primitive) + ) action = await unit.run_action(primitive, **params) uuid = action.id await model.disconnect() except Exception as e: - self.log.debug("Caught exception while executing primitive: {}".format(e)) + self.log.debug( + "Caught exception while executing primitive: {}".format(e) + ) raise N2VCPrimitiveExecutionFailed(e) return uuid diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_single_vdu_proxy_charm.py b/tests/test_single_vdu_proxy_charm.py new file mode 100644 index 0000000..a971872 --- /dev/null +++ b/tests/test_single_vdu_proxy_charm.py @@ -0,0 +1,351 @@ +"""Test the deployment and configuration of a proxy charm. + 1. Deploy proxy charm to a unit + 2. Execute 'get-ssh-public-key' primitive and get returned value + 3. Create LXD container with unit's public ssh key + 4. Verify SSH works between unit and container + 5. Destroy Juju unit + 6. Stop and Destroy LXD container +""" +import asyncio +import functools +import os +import sys +import logging +import unittest +from . import utils +import yaml +from n2vc.vnf import N2VC + +NSD_YAML = """ +nsd:nsd-catalog: + nsd: + - id: singlecharmvdu-ns + name: singlecharmvdu-ns + short-name: singlecharmvdu-ns + description: NS with 1 VNFs singlecharmvdu-vnf connected by datanet and mgmtnet VLs + version: '1.0' + logo: osm.png + constituent-vnfd: + - vnfd-id-ref: singlecharmvdu-vnf + member-vnf-index: '1' + vld: + - id: mgmtnet + name: mgmtnet + short-name: mgmtnet + type: ELAN + mgmt-network: 'true' + vim-network-name: mgmt + vnfd-connection-point-ref: + - vnfd-id-ref: singlecharmvdu-vnf + member-vnf-index-ref: '1' + vnfd-connection-point-ref: vnf-mgmt + - vnfd-id-ref: singlecharmvdu-vnf + member-vnf-index-ref: '2' + vnfd-connection-point-ref: vnf-mgmt + - id: datanet + name: datanet + short-name: datanet + type: ELAN + vnfd-connection-point-ref: + - vnfd-id-ref: singlecharmvdu-vnf + member-vnf-index-ref: '1' + vnfd-connection-point-ref: vnf-data + - vnfd-id-ref: singlecharmvdu-vnf + member-vnf-index-ref: '2' + vnfd-connection-point-ref: vnf-data +""" + +VNFD_YAML = """ +vnfd:vnfd-catalog: + vnfd: + - id: singlecharmvdu-vnf + name: singlecharmvdu-vnf + short-name: singlecharmvdu-vnf + version: '1.0' + description: A VNF consisting of 2 VDUs w/charms connected to an internal VL, and one VDU with cloud-init + logo: osm.png + connection-point: + - id: vnf-mgmt + name: vnf-mgmt + short-name: vnf-mgmt + type: VPORT + - id: vnf-data + name: vnf-data + short-name: vnf-data + type: VPORT + mgmt-interface: + cp: vnf-mgmt + internal-vld: + - id: internal + name: internal + short-name: internal + type: ELAN + internal-connection-point: + - id-ref: mgmtVM-internal + - id-ref: dataVM-internal + vdu: + - id: mgmtVM + name: mgmtVM + image: xenial + count: '1' + vm-flavor: + vcpu-count: '1' + memory-mb: '1024' + storage-gb: '10' + interface: + - name: mgmtVM-eth0 + position: '1' + type: EXTERNAL + virtual-interface: + type: VIRTIO + external-connection-point-ref: vnf-mgmt + - name: mgmtVM-eth1 + position: '2' + type: INTERNAL + virtual-interface: + type: VIRTIO + internal-connection-point-ref: mgmtVM-internal + internal-connection-point: + - id: mgmtVM-internal + name: mgmtVM-internal + short-name: mgmtVM-internal + type: VPORT + cloud-init-file: cloud-config.txt + vdu-configuration: + juju: + charm: simple + initial-config-primitive: + - seq: '1' + name: config + parameter: + - name: ssh-hostname + value: + - name: ssh-username + value: ubuntu + - name: ssh-password + value: ubuntu + - seq: '2' + name: touch + parameter: + - name: filename + value: '/home/ubuntu/first-touch-mgmtVM' + config-primitive: + - name: touch + parameter: + - name: filename + data-type: STRING + default-value: '/home/ubuntu/touched' + +""" + + +class PythonTest(unittest.TestCase): + n2vc = None + container = None + + def setUp(self): + self.log = logging.getLogger() + self.log.level = logging.DEBUG + + self.loop = asyncio.get_event_loop() + + # self.container = utils.create_lxd_container() + self.n2vc = utils.get_n2vc() + + def tearDown(self): + if self.container: + self.container.stop() + self.container.delete() + + self.loop.run_until_complete(self.n2vc.logout()) + + def n2vc_callback(self, model_name, application_name, workload_status, workload_message, task=None): + """We pass the vnfd when setting up the callback, so expect it to be + returned as a tuple.""" + self.log.debug("[Callback] Workload status '{}' for application {}".format(workload_status, application_name)) + self.log.debug("[Callback] Task: \"{}\"".format(task)) + + if workload_status == "exec_primitive" and task: + self.log.debug("Getting Primitive Status") + # get the uuid from the task + uuid = task.result() + + # get the status of the action + task = asyncio.ensure_future( + self.n2vc.GetPrimitiveStatus( + model_name, + uuid, + ) + ) + task.add_done_callback(functools.partial(self.n2vc_callback, model_name, application_name, "primitive_status", task)) + + if workload_status == "primitive_status" and task and not self.container: + self.log.debug("Creating LXD container") + # Get the ssh key + result = task.result() + pubkey = result['pubkey'] + + self.container = utils.create_lxd_container(pubkey) + mgmtaddr = self.container.state().network['eth0']['addresses'] + + self.log.debug("Setting config ssh-hostname={}".format(mgmtaddr[0]['address'])) + task = asyncio.ensure_future( + self.n2vc.ExecutePrimitive( + model_name, + application_name, + "config", + None, + params={ + 'ssh-hostname': mgmtaddr[0]['address'], + } + ) + ) + task.add_done_callback(functools.partial(self.n2vc_callback, model_name, application_name, None, None)) + + if workload_status and not task: + self.log.debug("Callback: workload status \"{}\"".format(workload_status)) + + if workload_status in ["blocked"] and not self.container: + self.log.debug("Getting public SSH key") + + # Execute 'get-ssh-public-key' primitive and get returned value + task = asyncio.ensure_future( + self.n2vc.ExecutePrimitive( + model_name, + application_name, + "get-ssh-public-key", + None, + params={ + 'ssh-hostname': '10.195.8.78', + 'ssh-username': 'ubuntu', + 'ssh-password': 'ubuntu' + } + ) + ) + task.add_done_callback(functools.partial(self.n2vc_callback, model_name, application_name, "exec_primitive", task)) + + + # task = asyncio.ensure_future( + # self.n2vc.ExecutePrimitive( + # model_name, + # application_name, + # "config", + # None, + # params={ + # 'ssh-hostname': '10.195.8.78', + # 'ssh-username': 'ubuntu', + # 'ssh-password': 'ubuntu' + # } + # ) + # ) + # task.add_done_callback(functools.partial(self.n2vc_callback, None, None, None)) + pass + elif workload_status in ["active"]: + self.log.debug("Removing charm") + task = asyncio.ensure_future( + self.n2vc.RemoveCharms(model_name, application_name, self.n2vc_callback) + ) + task.add_done_callback(functools.partial(self.n2vc_callback, None, None, None)) + + if self.container: + utils.destroy_lxd_container(self.container) + self.container = None + + # Stop the test + self.loop.call_soon_threadsafe(self.loop.stop) + + def test_deploy_application(self): + """Deploy proxy charm to a unit.""" + stream_handler = logging.StreamHandler(sys.stdout) + self.log.addHandler(stream_handler) + try: + self.log.info("Log handler installed") + nsd = utils.get_descriptor(NSD_YAML) + vnfd = utils.get_descriptor(VNFD_YAML) + + if nsd and vnfd: + + vca_charms = os.getenv('VCA_CHARMS', None) + + params = {} + vnf_index = 0 + + def deploy(): + """An inner function to do the deployment of a charm from + either a vdu or vnf. + """ + charm_dir = "{}/{}".format(vca_charms, charm) + + # Setting this to an IP that will fail the initial config. + # This will be detected in the callback, which will execute + # the "config" primitive with the right IP address. + # mgmtaddr = self.container.state().network['eth0']['addresses'] + # params['rw_mgmt_ip'] = mgmtaddr[0]['address'] + + # Legacy method is to set the ssh-private-key config + # with open(utils.get_juju_private_key(), "r") as f: + # pkey = f.readline() + # params['ssh-private-key'] = pkey + + ns_name = "default" + + vnf_name = self.n2vc.FormatApplicationName( + ns_name, + vnfd['name'], + str(vnf_index), + ) + + self.loop.run_until_complete( + self.n2vc.DeployCharms( + ns_name, + vnf_name, + vnfd, + charm_dir, + params, + {}, + self.n2vc_callback + ) + ) + + # Check if the VDUs in this VNF have a charm + for vdu in vnfd['vdu']: + vdu_config = vdu.get('vdu-configuration') + if vdu_config: + juju = vdu_config['juju'] + self.assertIsNotNone(juju) + + charm = juju['charm'] + self.assertIsNotNone(charm) + + params['initial-config-primitive'] = vdu_config['initial-config-primitive'] + + deploy() + vnf_index += 1 + + # Check if this VNF has a charm + vnf_config = vnfd.get("vnf-configuration") + if vnf_config: + juju = vnf_config['juju'] + self.assertIsNotNone(juju) + + charm = juju['charm'] + self.assertIsNotNone(charm) + + params['initial-config-primitive'] = vnf_config['initial-config-primitive'] + + deploy() + vnf_index += 1 + + self.loop.run_forever() + # while self.loop.is_running(): + # # await asyncio.sleep(1) + # time.sleep(1) + + # Test actions + # ExecutePrimitive(self, nsd, vnfd, vnf_member_index, primitive, callback, *callback_args, **params): + + # self.loop.run_until_complete(n.DestroyNetworkService(nsd)) + + # self.loop.run_until_complete(self.n2vc.logout()) + finally: + self.log.removeHandler(stream_handler) diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..9f9000e --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 + +import logging +import n2vc.vnf +import pylxd +import os +import time +import uuid +import yaml + +# Disable InsecureRequestWarning w/LXD +import urllib3 +urllib3.disable_warnings() + + +def get_descriptor(descriptor): + desc = None + try: + tmp = yaml.load(descriptor) + + # Remove the envelope + root = list(tmp.keys())[0] + if root == "nsd:nsd-catalog": + desc = tmp['nsd:nsd-catalog']['nsd'][0] + elif root == "vnfd:vnfd-catalog": + desc = tmp['vnfd:vnfd-catalog']['vnfd'][0] + except ValueError: + assert False + return desc + +def get_n2vc(): + """Return an instance of N2VC.VNF.""" + log = logging.getLogger() + log.level = logging.DEBUG + + # Extract parameters from the environment in order to run our test + vca_host = os.getenv('VCA_HOST', '127.0.0.1') + vca_port = os.getenv('VCA_PORT', 17070) + vca_user = os.getenv('VCA_USER', 'admin') + vca_charms = os.getenv('VCA_CHARMS', None) + vca_secret = os.getenv('VCA_SECRET', None) + client = n2vc.vnf.N2VC( + log=log, + server=vca_host, + port=vca_port, + user=vca_user, + secret=vca_secret, + artifacts=vca_charms, + ) + return client + +def create_lxd_container(public_key=None): + """ + Returns a container object + + If public_key isn't set, we'll use the Juju ssh key + """ + + client = get_lxd_client() + test_machine = "test-{}-add-manual-machine-ssh".format( + uuid.uuid4().hex[-4:] + ) + + private_key_path, public_key_path = find_juju_ssh_keys() + # private_key_path = os.path.expanduser( + # "~/.local/share/juju/ssh/juju_id_rsa" + # ) + # public_key_path = os.path.expanduser( + # "~/.local/share/juju/ssh/juju_id_rsa.pub" + # ) + + # Use the self-signed cert generated by lxc on first run + crt = os.path.expanduser('~/snap/lxd/current/.config/lxc/client.crt') + assert os.path.exists(crt) + + key = os.path.expanduser('~/snap/lxd/current/.config/lxc/client.key') + assert os.path.exists(key) + + # create profile w/cloud-init and juju ssh key + if not public_key: + public_key = "" + with open(public_key_path, "r") as f: + public_key = f.readline() + + profile = client.profiles.create( + test_machine, + config={'user.user-data': '#cloud-config\nssh_authorized_keys:\n- {}'.format(public_key)}, + devices={ + 'root': {'path': '/', 'pool': 'default', 'type': 'disk'}, + 'eth0': { + 'nictype': 'bridged', + 'parent': 'lxdbr0', + 'type': 'nic' + } + } + ) + + # create lxc machine + config = { + 'name': test_machine, + 'source': { + 'type': 'image', + 'alias': 'xenial', + 'mode': 'pull', + 'protocol': 'simplestreams', + 'server': 'https://cloud-images.ubuntu.com/releases', + }, + 'profiles': [test_machine], + } + container = client.containers.create(config, wait=True) + container.start(wait=True) + + def wait_for_network(container, timeout=30): + """Wait for eth0 to have an ipv4 address.""" + starttime = time.time() + while(time.time() < starttime + timeout): + time.sleep(1) + if 'eth0' in container.state().network: + addresses = container.state().network['eth0']['addresses'] + if len(addresses) > 0: + if addresses[0]['family'] == 'inet': + return addresses[0] + return None + + host = wait_for_network(container) + + # HACK: We need to give sshd a chance to bind to the interface, + # and pylxd's container.execute seems to be broken and fails and/or + # hangs trying to properly check if the service is up. + time.sleep(5) + + return container + + +def destroy_lxd_container(container): + """Stop and delete a LXD container.""" + container.stop(wait=True) + container.delete() + + +def find_lxd_config(): + """Find the LXD configuration directory.""" + paths = [] + paths.append(os.path.expanduser("~/.config/lxc")) + paths.append(os.path.expanduser("~/snap/lxd/current/.config/lxc")) + + for path in paths: + if os.path.exists(path): + crt = os.path.expanduser("{}/client.crt".format(path)) + key = os.path.expanduser("{}/client.key".format(path)) + if os.path.exists(crt) and os.path.exists(key): + return (crt, key) + return (None, None) + + +def find_juju_ssh_keys(): + """Find the Juju ssh keys.""" + + paths = [] + paths.append(os.path.expanduser("~/.local/share/juju/ssh/")) + + for path in paths: + if os.path.exists(path): + private = os.path.expanduser("{}/juju_id_rsa".format(path)) + public = os.path.expanduser("{}/juju_id_rsa.pub".format(path)) + if os.path.exists(private) and os.path.exists(public): + return (private, public) + return (None, None) + + +def get_juju_private_key(): + keys = find_juju_ssh_keys() + return keys[0] + + +def get_lxd_client(host="127.0.0.1", port="8443", verify=False): + """ Get the LXD client.""" + client = None + (crt, key) = find_lxd_config() + + if crt and key: + client = pylxd.Client( + endpoint="https://{}:{}".format(host, port), + cert=(crt, key), + verify=verify, + ) + + return client