From c3e6c2ec9a1fddfc8e9bd31509b366e633b6d99e Mon Sep 17 00:00:00 2001 From: Adam Israel Date: Thu, 1 Mar 2018 09:31:50 -0500 Subject: [PATCH] New N2VC interface + updated libjuju This commit introduces the Python3 N2VC module, which acts as a standard interface to the VCA. The goal of this is to provide a common way for modules to interface with the VCA. - Updated libjuju from 0.6.1 to 0.7.3 Signed-off-by: Adam Israel Change-Id: Ide70fb5ae5797eb6486de24653dc09a23f9c009e --- .gitignore | 2 + README.md | 148 +++- examples/credential.py | 46 -- modules/libjuju/.travis.yml | 29 +- modules/libjuju/Makefile | 8 +- modules/libjuju/VERSION | 2 +- modules/libjuju/docs/api/juju.client.rst | 4 +- modules/libjuju/docs/api/modules.rst | 4 +- modules/libjuju/docs/changelog.rst | 53 +- modules/libjuju/docs/index.rst | 21 +- modules/libjuju/docs/narrative/index.rst | 3 +- modules/libjuju/docs/readme.rst | 36 +- modules/libjuju/docs/requirements.txt | 8 +- .../libjuju/docs/upstream-updates/index.rst | 4 +- modules/libjuju/examples/action.py | 2 +- modules/libjuju/examples/add_machine.py | 2 +- modules/libjuju/examples/add_model.py | 2 +- modules/libjuju/examples/allwatcher.py | 2 +- modules/libjuju/examples/config.py | 2 +- modules/libjuju/examples/controller.py | 2 +- modules/libjuju/examples/deploy.py | 2 +- modules/libjuju/examples/fullstatus.py | 2 +- modules/libjuju/examples/future.py | 2 +- modules/libjuju/examples/leadership.py | 2 +- modules/libjuju/examples/livemodel.py | 2 +- modules/libjuju/examples/localcharm.py | 2 +- modules/libjuju/examples/relate.py | 99 +-- modules/libjuju/examples/unitrun.py | 2 +- modules/libjuju/juju/application.py | 18 + modules/libjuju/juju/client/_client.py | 12 +- modules/libjuju/juju/client/_client1.py | 4 +- modules/libjuju/juju/client/_client2.py | 4 +- modules/libjuju/juju/client/_client3.py | 4 +- modules/libjuju/juju/client/_client4.py | 4 +- modules/libjuju/juju/client/_client5.py | 4 +- modules/libjuju/juju/client/_definitions.py | 4 +- modules/libjuju/juju/client/client.py | 5 +- modules/libjuju/juju/client/connection.py | 634 +++++++--------- modules/libjuju/juju/client/connector.py | 147 ++++ modules/libjuju/juju/client/facade.py | 8 +- modules/libjuju/juju/client/gocookies.py | 102 +++ modules/libjuju/juju/client/jujudata.py | 219 ++++++ modules/libjuju/juju/client/overrides.py | 6 +- modules/libjuju/juju/controller.py | 284 +++++--- modules/libjuju/juju/errors.py | 22 + modules/libjuju/juju/machine.py | 19 +- modules/libjuju/juju/model.py | 373 ++++++---- modules/libjuju/juju/relation.py | 112 +++ modules/libjuju/juju/tag.py | 5 + modules/libjuju/juju/unit.py | 6 +- {juju => modules/libjuju/juju}/user.py | 9 +- modules/libjuju/juju/utils.py | 15 + modules/libjuju/tests/base.py | 121 +-- .../libjuju/tests/integration/test_client.py | 6 +- .../tests/integration/test_connection.py | 48 +- .../tests/integration/test_controller.py | 36 +- .../libjuju/tests/integration/test_errors.py | 6 +- .../libjuju/tests/integration/test_machine.py | 9 +- .../libjuju/tests/integration/test_model.py | 79 +- .../libjuju/tests/integration/test_unit.py | 9 +- modules/libjuju/tests/unit/test_client.py | 1 - modules/libjuju/tests/unit/test_connection.py | 27 +- .../libjuju/tests/unit/test_constraints.py | 1 + modules/libjuju/tests/unit/test_loop.py | 2 + modules/libjuju/tests/unit/test_model.py | 63 +- modules/libjuju/tests/unit/test_overrides.py | 4 +- modules/libjuju/tests/unit/test_placement.py | 2 +- n2vc/__init__.py | 1 + n2vc/vnf.py | 686 ++++++++++++++++++ requirements.txt | 1 + setup.py | 40 + tests/test_async_task.py | 16 + tests/test_python.py | 434 +++++++++++ tox.ini | 25 + 74 files changed, 3183 insertions(+), 947 deletions(-) create mode 100644 .gitignore delete mode 100644 examples/credential.py create mode 100644 modules/libjuju/juju/client/connector.py create mode 100644 modules/libjuju/juju/client/gocookies.py create mode 100644 modules/libjuju/juju/client/jujudata.py rename {juju => modules/libjuju/juju}/user.py (90%) create mode 100644 n2vc/__init__.py create mode 100644 n2vc/vnf.py create mode 100644 requirements.txt create mode 100644 setup.py create mode 100644 tests/test_async_task.py create mode 100755 tests/test_python.py create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8d35cb3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__ +*.pyc diff --git a/README.md b/README.md index 664d412..a4cfb9d 100644 --- a/README.md +++ b/README.md @@ -2,4 +2,150 @@ ## Objective -N2VC aims to be a generic VNF configuration and management (GVNFM) library, allowing OSM modules to interact with any supported GVNFM system. +The N2VC library provides an OSM-centric interface to the VCA. This enables any OSM module (SO, LCM, osmclient) to use a standard pattern for communicating with Juju and the charms responsible for configuring a VNF. + +N2VC relies on the IM module, enforcing compliance with the OSM Information Model. + +## Caveats + +This library is in active development for OSM Release FOUR. The interface is subject to change prior to release. + +## Roadmap +- Create the N2VC API (in progress) +- Create a Python library implementing the N2VC API (in progress) +- Implement N2VC API in SO +- Implement N2VC API in lcm +- Add support for N2VC in OSMClient + +## Requirements + +Because this is still in heavy development, there are a few manual steps required to use this library. + + +``` +# This list is incomplete +apt install python3-nose +``` + +### Install LXD and Juju + +In order to run the test(s) included in N2VC, you'll need to install Juju locally. + +*Note: It's not necessary to install the juju library via pip3; N2VC uses a version bundled within the modules/ directory.* + +```bash +snap install lxd +snap install juju --classic +``` + +### Install the IM + +The pyangbind library, used by the IM to parse the Yang data models does not support Python3 at the moment. Work is being done in coordination with the pyangbind maintainer, and Python3 support is expected to hand in the near future. + +In the meantime, we will install and use Python3-compatible branches of pyangbind and the IM module. + +``` +sudo apt install libxml2-dev libxslt1-dev build-essential + +git clone http://github.com/adamisrael/pyangbind.git +cd pyangbind +pip3 install -r requirements.txt +pip3 install -r requirements.DEVELOPER.txt + +sudo python3 setup.py develop +``` + +Checkout the IM module and use the proposed branch from Gerrit: +```bash +git clone https://osm.etsi.org/gerrit/osm/IM.git +cd IM +git fetch https://israelad@osm.etsi.org/gerrit/osm/IM refs/changes/78/5778/2 && git checkout FETCH_HEAD +sudo python3 setup.py develop +``` + +```bash +git clone https://osm.etsi.org/gerrit/osm/N2VC.git +cd N2VC +git fetch https://israelad@osm.etsi.org/gerrit/osm/N2VC refs/changes/70/5870/5 && git checkout FETCH_HEAD +sudo python3 setup.py develop + +``` + +## Testing +A basic test has been written to exercise the functionality of the library, and to serve as a demonstration of how to use it. + +### Export settings to run test + +Export a few environment variables so the test knows where to find the VCA, and the compiled pingpong charm from the devops repository. + +```bash +# You can find the ip of the VCA by running `juju status -m controller` and looking for the DNS for Machine 0 +export VCA_HOST= +export VCA_PORT=17070 +# You can find these variables in ~/.local/share/juju/accounts.yaml +export VCA_USER=admin +export VCA_SECRET=PASSWORD +``` + +### Run the test(s) + +*Note: There is a bug in the cleanup of the N2VC/Juju that will throw an exception after the test has finished. This is on the list of things to fix for R4 and should not impact your tests or integration.* + +```bash +nosetests3 --nocapture tests/test_python.py +``` + +## Known Issues + +Many. This is still in active development for Release FOUR. + +- `GetMetrics` does not work yet. There seems to be +- An exception is thrown after using N2VC, probably related to the internal AllWatcher used by juju.Model. This shouldn't break usage of N2VC, but it is ugly and needs to be fixed. + +``` +Exception ignored in: +Traceback (most recent call last): + File "/home/stone/.local/lib/python3.6/site-packages/websockets/protocol.py", line 743, in close_connection + if (yield from self.wait_for_connection_lost()): + File "/home/stone/.local/lib/python3.6/site-packages/websockets/protocol.py", line 768, in wait_for_connection_lost + self.timeout, loop=self.loop) + File "/usr/lib/python3.6/asyncio/tasks.py", line 342, in wait_for + timeout_handle = loop.call_later(timeout, _release_waiter, waiter) + File "/usr/lib/python3.6/asyncio/base_events.py", line 543, in call_later + timer = self.call_at(self.time() + delay, callback, *args) + File "/usr/lib/python3.6/asyncio/base_events.py", line 553, in call_at + self._check_closed() + File "/usr/lib/python3.6/asyncio/base_events.py", line 357, in _check_closed + raise RuntimeError('Event loop is closed') +RuntimeError: Event loop is closed +Exception ignored in: +Traceback (most recent call last): + File "/usr/lib/python3.6/asyncio/queues.py", line 169, in get + getter.cancel() # Just in case getter is not done yet. + File "/usr/lib/python3.6/asyncio/base_events.py", line 574, in call_soon + self._check_closed() + File "/usr/lib/python3.6/asyncio/base_events.py", line 357, in _check_closed + raise RuntimeError('Event loop is closed') +RuntimeError: Event loop is closed +Exception ignored in: +Traceback (most recent call last): + File "/home/stone/src/osm/N2VC/modules/libjuju/juju/client/facade.py", line 412, in wrapper + reply = await f(*args, **kwargs) + File "/home/stone/src/osm/N2VC/modules/libjuju/juju/client/_client1.py", line 59, in Next + reply = await self.rpc(msg) + File "/home/stone/src/osm/N2VC/modules/libjuju/juju/client/overrides.py", line 104, in rpc + result = await self.connection.rpc(msg, encoder=TypeEncoder) + File "/home/stone/src/osm/N2VC/modules/libjuju/juju/client/connection.py", line 306, in rpc + result = await self._recv(msg['request-id']) + File "/home/stone/src/osm/N2VC/modules/libjuju/juju/client/connection.py", line 208, in _recv + return await self.messages.get(request_id) + File "/home/stone/src/osm/N2VC/modules/libjuju/juju/utils.py", line 61, in get + value = await self._queues[id].get() + File "/usr/lib/python3.6/asyncio/queues.py", line 169, in get + getter.cancel() # Just in case getter is not done yet. + File "/usr/lib/python3.6/asyncio/base_events.py", line 574, in call_soon + self._check_closed() + File "/usr/lib/python3.6/asyncio/base_events.py", line 357, in _check_closed + raise RuntimeError('Event loop is closed') +RuntimeError: Event loop is closed +``` diff --git a/examples/credential.py b/examples/credential.py deleted file mode 100644 index f335af9..0000000 --- a/examples/credential.py +++ /dev/null @@ -1,46 +0,0 @@ -import sys -from juju import loop -from juju.controller import Controller - - -async def main(cloud_name, credential_name): - controller = Controller() - model = None - print('Connecting to controller') - await controller.connect_current() - try: - print('Adding model') - model = await controller.add_model( - 'test', - cloud_name=cloud_name, - credential_name=credential_name) - - # verify credential - print("Verify model's credential: {}".format( - model.info.cloud_credential_tag)) - - # verify we can deploy - print('Deploying ubuntu') - app = await model.deploy('ubuntu-10') - - print('Waiting for active') - await model.block_until( - lambda: app.units and all(unit.workload_status == 'active' - for unit in app.units)) - - print('Removing ubuntu') - await app.remove() - finally: - print('Cleaning up') - if model: - print('Removing model') - model_uuid = model.info.uuid - await model.disconnect() - await controller.destroy_model(model_uuid) - print('Disconnecting') - await controller.disconnect() - - -if __name__ == '__main__': - assert len(sys.argv) > 2, 'Please provide a cloud and credential name' - loop.run(main(sys.argv[1], sys.argv[2])) diff --git a/modules/libjuju/.travis.yml b/modules/libjuju/.travis.yml index 16e88dc..0e907f0 100644 --- a/modules/libjuju/.travis.yml +++ b/modules/libjuju/.travis.yml @@ -2,20 +2,29 @@ dist: trusty sudo: required language: python python: - - "3.5" + - "3.6" +before_script: + - sudo addgroup lxd || true + - sudo usermod -a -G lxd $USER || true + - sudo ln -s /snap/bin/juju /usr/bin/juju + - sudo ln -s /snap/bin/lxc /usr/bin/lxc before_install: - - sudo add-apt-repository ppa:ubuntu-lxc/lxd-stable -y + - sudo add-apt-repository -y ppa:jonathonf/python-3.6 + - sudo add-apt-repository ppa:chris-lea/libsodium -y - sudo apt-get update -q - - sudo apt-get install lxd snapd -y - - sudo usermod -a -G lxd $USER - - sudo service lxd start || true - - sudo lxd init --auto + - sudo apt-get remove -qy lxd lxd-client + - sudo apt-get install snapd libsodium-dev -y + - sudo snap install lxd || true + - sudo snap install juju-wait --classic || true install: pip install tox-travis env: - - SNAP_CMD="sudo snap install juju --classic --stable" - - SNAP_CMD="sudo snap install juju --classic --edge" + global: > + TEST_AGENTS='{"agents":[{"url":"https://api.staging.jujucharms.com/identity","username":"libjuju-ci@yellow"}],"key":{"private":"88OOCxIHQNguRG7zFg2y2Hx5Ob0SeVKKBRnjyehverc=","public":"fDn20+5FGyN2hYO7z0rFUyoHGUnfrleslUNtoYsjNSs="}}' + matrix: + - JUJU_CHANNEL=stable + - JUJU_CHANNEL=edge script: - - (eval "$SNAP_CMD") + - sudo snap install juju --classic --$JUJU_CHANNEL - sudo ln -s /snap/bin/juju /usr/bin/juju || true - - sudo -E sudo -u $USER -E bash -c "/snap/bin/juju bootstrap localhost test" + - sudo -E sudo -u $USER -E /snap/bin/juju bootstrap localhost test --config 'identity-url=https://api.staging.jujucharms.com/identity' --config 'allow-model-access=true' - tox -e py35,integration diff --git a/modules/libjuju/Makefile b/modules/libjuju/Makefile index 2e59306..bd59a97 100644 --- a/modules/libjuju/Makefile +++ b/modules/libjuju/Makefile @@ -1,5 +1,5 @@ -BIN := .tox/py35/bin -PY := $(BIN)/python3.5 +BIN := .tox/py3/bin +PY := $(BIN)/python PIP := $(BIN)/pip SCHEMAGEN := $(shell which schemagen) VERSION=$(shell cat VERSION) @@ -29,10 +29,10 @@ docs: .tox cd docs/_build/ && zip -r docs.zip * release: - git remote | xargs -L1 git fetch --tags + git fetch --tags $(PY) setup.py sdist upload git tag ${VERSION} - git remote | xargs -L1 git push --tags + git push --tags upload: release diff --git a/modules/libjuju/VERSION b/modules/libjuju/VERSION index ee6cdce..f38fc53 100644 --- a/modules/libjuju/VERSION +++ b/modules/libjuju/VERSION @@ -1 +1 @@ -0.6.1 +0.7.3 diff --git a/modules/libjuju/docs/api/juju.client.rst b/modules/libjuju/docs/api/juju.client.rst index 6a699c0..dad691f 100644 --- a/modules/libjuju/docs/api/juju.client.rst +++ b/modules/libjuju/docs/api/juju.client.rst @@ -1,5 +1,5 @@ -juju\.client package -==================== +Internal APIs +============= These packages are for internal use in communicating with the low-level API. You should use the object oriented API instead. They are documented diff --git a/modules/libjuju/docs/api/modules.rst b/modules/libjuju/docs/api/modules.rst index bf06f26..9722a6a 100644 --- a/modules/libjuju/docs/api/modules.rst +++ b/modules/libjuju/docs/api/modules.rst @@ -1,5 +1,5 @@ -juju -==== +Public APIs +=========== It is recommended that you start with :doc:`juju.model` or :doc:`juju.controller`. If you need helpers to manage the asyncio loop, try :doc:`juju.loop`. diff --git a/modules/libjuju/docs/changelog.rst b/modules/libjuju/docs/changelog.rst index d3d2e91..caf778e 100644 --- a/modules/libjuju/docs/changelog.rst +++ b/modules/libjuju/docs/changelog.rst @@ -1,5 +1,54 @@ -Change Log ----------- +Changelog +--------- + +0.7.3 +^^^^^ +Tuesday Feb 20 2018 + +* Full macaroon bakery support (#206) +* Fix regression with deploying local charm, add test case (#209) +* Expose a machines series (#208) +* Automated test runner fixes (#205) + +0.7.2 +^^^^^ +Friday Feb 9 2018 + +* Support deploying bundle YAML file directly (rather than just directory) (#202) + +0.7.1 +^^^^^ +Monday Dec 18 2017 + +* Fix missed renames of model_uuids (#197) + +0.7.0 +^^^^^ +Fri Dec 15 2017 + +* Fix race condition in adding relations (#192) +* Fix race condition in connection monitor test (#183) +* Fix example in README (#178) +* Fix rare hang during Unit.run (#177) +* Fix licensing quirks (#176) +* Refactor model handling (#171) +* Refactor users handling, add get_users (#170) +* Upload credential to controller when adding model (#168) +* Support 'applications' key in bundles (#165) +* Improve handling of thread error handling for loop.run() (#169) +* Fix encoding when using to_json() (#166) +* Fix intermittent test failures (#167) + +0.6.1 +^^^^^ +Fri Sept 29 2017 + +* Fix failure when controller supports newer facade version (#145) +* Fix test failures (#163) +* Fix SSH key handling when adding a new model (#161) +* Make Application.upgrade_charm upgrade resources (#158) +* Expand integration tests to use stable/edge versions of juju (#155) +* Move docs to ReadTheDocs (https://pythonlibjuju.readthedocs.io/en/latest/) 0.6.1 ^^^^^ diff --git a/modules/libjuju/docs/index.rst b/modules/libjuju/docs/index.rst index b4b075f..2dd55cb 100644 --- a/modules/libjuju/docs/index.rst +++ b/modules/libjuju/docs/index.rst @@ -11,16 +11,29 @@ Table of Contents ----------------- .. toctree:: + :caption: Overview :glob: :maxdepth: 3 narrative/index - API Docs - Internal API Docs - upstream-updates/index -.. include:: changelog.rst +.. toctree:: + :caption: API Documentation + :glob: + :maxdepth: 3 + + api/modules + api/juju.client + +.. toctree:: + :caption: Project + :glob: + :maxdepth: 3 + + upstream-updates/index + changelog + Indices and tables diff --git a/modules/libjuju/docs/narrative/index.rst b/modules/libjuju/docs/narrative/index.rst index eb77e4c..b1684a0 100644 --- a/modules/libjuju/docs/narrative/index.rst +++ b/modules/libjuju/docs/narrative/index.rst @@ -1,5 +1,4 @@ -Narrative Docs -============== +**Overview** .. toctree:: :glob: diff --git a/modules/libjuju/docs/readme.rst b/modules/libjuju/docs/readme.rst index ecfbc5a..886550d 100644 --- a/modules/libjuju/docs/readme.rst +++ b/modules/libjuju/docs/readme.rst @@ -46,10 +46,10 @@ and in the documentation. .. code:: python - #!/usr/bin/python3.5 + #!/usr/bin/python3 - import asyncio import logging + import sys from juju import loop from juju.model import Model @@ -63,26 +63,28 @@ and in the documentation. # Connect to the currently active Juju model await model.connect_current() - # Deploy a single unit of the ubuntu charm, using revision 0 from the - # stable channel of the Charm Store. - ubuntu_app = await model.deploy( - 'ubuntu-0', - application_name='ubuntu', - series='xenial', - channel='stable', - ) + try: + # Deploy a single unit of the ubuntu charm, using the latest revision + # from the stable channel of the Charm Store. + ubuntu_app = await model.deploy( + 'ubuntu', + application_name='ubuntu', + series='xenial', + channel='stable', + ) - # Disconnect from the api server and cleanup. - model.disconnect() + if '--wait' in sys.argv: + # optionally block until the application is ready + await model.block_until(lambda: ubuntu_app.status == 'active') + finally: + # Disconnect from the api server and cleanup. + await model.disconnect() def main(): - # Set logging level to debug so we can see verbose output from the - # juju library. - logging.basicConfig(level=logging.DEBUG) + logging.basicConfig(level=logging.INFO) - # Quiet logging from the websocket library. If you want to see - # everything sent over the wire, set this to DEBUG. + # If you want to see everything sent over the wire, set this to DEBUG. ws_logger = logging.getLogger('websockets.protocol') ws_logger.setLevel(logging.INFO) diff --git a/modules/libjuju/docs/requirements.txt b/modules/libjuju/docs/requirements.txt index 06377bf..dabf3a0 100644 --- a/modules/libjuju/docs/requirements.txt +++ b/modules/libjuju/docs/requirements.txt @@ -1,7 +1,5 @@ -websockets -pyyaml -theblues -python-dateutil -sphinx +pytz<2018.0,>=2017.2 # conflict between sphinx and macaroonbakery +pymacaroons>=0.13.0,<1.0 # force new version with pynacl instead of libnacl +sphinx==1.6.5 sphinxcontrib-asyncio sphinx_rtd_theme diff --git a/modules/libjuju/docs/upstream-updates/index.rst b/modules/libjuju/docs/upstream-updates/index.rst index 52099e6..7082a6e 100644 --- a/modules/libjuju/docs/upstream-updates/index.rst +++ b/modules/libjuju/docs/upstream-updates/index.rst @@ -1,5 +1,5 @@ -Upstream Updates -================ +Syncing Upstream Updates +======================== Updating the facade and definitions code generated from the schema to reflect changes in upstream Juju consists of two steps: diff --git a/modules/libjuju/examples/action.py b/modules/libjuju/examples/action.py index 0f25647..4a3cc6d 100644 --- a/modules/libjuju/examples/action.py +++ b/modules/libjuju/examples/action.py @@ -27,7 +27,7 @@ async def run_action(unit): async def main(): model = Model() - await model.connect_current() + await model.connect() await model.reset(force=True) app = await model.deploy( diff --git a/modules/libjuju/examples/add_machine.py b/modules/libjuju/examples/add_machine.py index 8ae2d40..33d0c34 100755 --- a/modules/libjuju/examples/add_machine.py +++ b/modules/libjuju/examples/add_machine.py @@ -19,7 +19,7 @@ GB = 1024 async def main(): model = Model() - await model.connect_current() + await model.connect() try: # add a new default machine diff --git a/modules/libjuju/examples/add_model.py b/modules/libjuju/examples/add_model.py index 3e46490..0e96fa1 100644 --- a/modules/libjuju/examples/add_model.py +++ b/modules/libjuju/examples/add_model.py @@ -19,7 +19,7 @@ LOG = getLogger(__name__) async def main(): controller = Controller() print("Connecting to controller") - await controller.connect_current() + await controller.connect() try: model_name = "addmodeltest-{}".format(uuid.uuid4()) diff --git a/modules/libjuju/examples/allwatcher.py b/modules/libjuju/examples/allwatcher.py index c78d689..884230b 100644 --- a/modules/libjuju/examples/allwatcher.py +++ b/modules/libjuju/examples/allwatcher.py @@ -16,7 +16,7 @@ from juju import loop async def watch(): - conn = await Connection.connect_current() + conn = await Connection.connect() allwatcher = client.AllWatcherFacade.from_connection(conn) while True: change = await allwatcher.Next() diff --git a/modules/libjuju/examples/config.py b/modules/libjuju/examples/config.py index bacc840..bad5b6d 100644 --- a/modules/libjuju/examples/config.py +++ b/modules/libjuju/examples/config.py @@ -19,7 +19,7 @@ MB = 1 async def main(): model = Model() - await model.connect_current() + await model.connect() await model.reset(force=True) ubuntu_app = await model.deploy( diff --git a/modules/libjuju/examples/controller.py b/modules/libjuju/examples/controller.py index 6002f68..3f029ab 100644 --- a/modules/libjuju/examples/controller.py +++ b/modules/libjuju/examples/controller.py @@ -17,7 +17,7 @@ from juju import loop async def main(): controller = Controller() - await controller.connect_current() + await controller.connect() model = await controller.add_model( 'my-test-model', 'aws', diff --git a/modules/libjuju/examples/deploy.py b/modules/libjuju/examples/deploy.py index e6c306a..b14e4ca 100644 --- a/modules/libjuju/examples/deploy.py +++ b/modules/libjuju/examples/deploy.py @@ -13,7 +13,7 @@ from juju.model import Model async def main(): model = Model() print('Connecting to model') - await model.connect_current() + await model.connect() try: print('Deploying ubuntu') diff --git a/modules/libjuju/examples/fullstatus.py b/modules/libjuju/examples/fullstatus.py index cdaf51d..5548423 100644 --- a/modules/libjuju/examples/fullstatus.py +++ b/modules/libjuju/examples/fullstatus.py @@ -5,7 +5,7 @@ from juju.client.client import ClientFacade from juju import loop async def status(): - conn = await Connection.connect_current() + conn = await Connection.connect() client = ClientFacade.from_connection(conn) patterns = None diff --git a/modules/libjuju/examples/future.py b/modules/libjuju/examples/future.py index 0180325..c93981a 100644 --- a/modules/libjuju/examples/future.py +++ b/modules/libjuju/examples/future.py @@ -11,7 +11,7 @@ from juju import loop async def main(): model = Model() - await model.connect_current() + await model.connect() await model.reset(force=True) goal_state = Model.from_yaml('bundle-like-thing') diff --git a/modules/libjuju/examples/leadership.py b/modules/libjuju/examples/leadership.py index b231003..dbd1b6e 100644 --- a/modules/libjuju/examples/leadership.py +++ b/modules/libjuju/examples/leadership.py @@ -13,7 +13,7 @@ from juju import loop async def report_leadership(): model = Model() - await model.connect_current() + await model.connect() print("Leadership: ") for app in model.applications.values(): diff --git a/modules/libjuju/examples/livemodel.py b/modules/libjuju/examples/livemodel.py index 47eb999..a15e9f7 100644 --- a/modules/libjuju/examples/livemodel.py +++ b/modules/libjuju/examples/livemodel.py @@ -21,7 +21,7 @@ async def on_model_change(delta, old, new, model): async def watch_model(): model = Model() - await model.connect_current() + await model.connect() model.add_observer(on_model_change) diff --git a/modules/libjuju/examples/localcharm.py b/modules/libjuju/examples/localcharm.py index 978703e..b9481d4 100644 --- a/modules/libjuju/examples/localcharm.py +++ b/modules/libjuju/examples/localcharm.py @@ -15,7 +15,7 @@ from juju import loop async def main(): model = Model() - await model.connect_current() + await model.connect() # Deploy a local charm using a path to the charm directory await model.deploy( diff --git a/modules/libjuju/examples/relate.py b/modules/libjuju/examples/relate.py index 8f1e708..c0ce4c6 100644 --- a/modules/libjuju/examples/relate.py +++ b/modules/libjuju/examples/relate.py @@ -40,59 +40,62 @@ class MyModelObserver(ModelObserver): async def main(): model = Model() - await model.connect_current() + await model.connect() - model.add_observer(MyRemoveObserver()) - await model.reset(force=True) - model.add_observer(MyModelObserver()) + try: + model.add_observer(MyRemoveObserver()) + await model.reset(force=True) + model.add_observer(MyModelObserver()) - ubuntu_app = await model.deploy( - 'ubuntu', - application_name='ubuntu', - series='trusty', - channel='stable', - ) - ubuntu_app.on_change(asyncio.coroutine( - lambda delta, old_app, new_app, model: - print('App changed: {}'.format(new_app.entity_id)) - )) - ubuntu_app.on_remove(asyncio.coroutine( - lambda delta, old_app, new_app, model: - print('App removed: {}'.format(old_app.entity_id)) - )) - ubuntu_app.on_unit_add(asyncio.coroutine( - lambda delta, old_unit, new_unit, model: - print('Unit added: {}'.format(new_unit.entity_id)) - )) - ubuntu_app.on_unit_remove(asyncio.coroutine( - lambda delta, old_unit, new_unit, model: - print('Unit removed: {}'.format(old_unit.entity_id)) - )) - unit_a, unit_b = await ubuntu_app.add_units(count=2) - unit_a.on_change(asyncio.coroutine( - lambda delta, old_unit, new_unit, model: - print('Unit changed: {}'.format(new_unit.entity_id)) - )) - await model.deploy( - 'nrpe', - application_name='nrpe', - series='trusty', - channel='stable', - # subordinates must be deployed without units - num_units=0, - ) - my_relation = await model.add_relation( - 'ubuntu', - 'nrpe', - ) - my_relation.on_remove(asyncio.coroutine( - lambda delta, old_rel, new_rel, model: - print('Relation removed: {}'.format(old_rel.endpoints)) - )) + ubuntu_app = await model.deploy( + 'ubuntu', + application_name='ubuntu', + series='trusty', + channel='stable', + ) + ubuntu_app.on_change(asyncio.coroutine( + lambda delta, old_app, new_app, model: + print('App changed: {}'.format(new_app.entity_id)) + )) + ubuntu_app.on_remove(asyncio.coroutine( + lambda delta, old_app, new_app, model: + print('App removed: {}'.format(old_app.entity_id)) + )) + ubuntu_app.on_unit_add(asyncio.coroutine( + lambda delta, old_unit, new_unit, model: + print('Unit added: {}'.format(new_unit.entity_id)) + )) + ubuntu_app.on_unit_remove(asyncio.coroutine( + lambda delta, old_unit, new_unit, model: + print('Unit removed: {}'.format(old_unit.entity_id)) + )) + unit_a, unit_b = await ubuntu_app.add_units(count=2) + unit_a.on_change(asyncio.coroutine( + lambda delta, old_unit, new_unit, model: + print('Unit changed: {}'.format(new_unit.entity_id)) + )) + await model.deploy( + 'nrpe', + application_name='nrpe', + series='trusty', + channel='stable', + # subordinates must be deployed without units + num_units=0, + ) + my_relation = await model.add_relation( + 'ubuntu', + 'nrpe', + ) + my_relation.on_remove(asyncio.coroutine( + lambda delta, old_rel, new_rel, model: + print('Relation removed: {}'.format(old_rel.endpoints)) + )) + finally: + await model.disconnect() if __name__ == '__main__': - logging.basicConfig(level=logging.DEBUG) + logging.basicConfig(level=logging.INFO) ws_logger = logging.getLogger('websockets.protocol') ws_logger.setLevel(logging.INFO) loop.run(main()) diff --git a/modules/libjuju/examples/unitrun.py b/modules/libjuju/examples/unitrun.py index 3dfacd6..b6e2240 100644 --- a/modules/libjuju/examples/unitrun.py +++ b/modules/libjuju/examples/unitrun.py @@ -24,7 +24,7 @@ async def run_command(unit): async def main(): model = Model() - await model.connect_current() + await model.connect() await model.reset(force=True) app = await model.deploy( diff --git a/modules/libjuju/juju/application.py b/modules/libjuju/juju/application.py index 620e9c9..555bb3d 100644 --- a/modules/libjuju/juju/application.py +++ b/modules/libjuju/juju/application.py @@ -51,6 +51,24 @@ class Application(model.ModelEntity): if unit.application == self.name ] + @property + def relations(self): + return [rel for rel in self.model.relations if rel.matches(self.name)] + + def related_applications(self, endpoint_name=None): + apps = {} + for rel in self.relations: + if rel.is_peer: + local_ep, remote_ep = rel.endpoints[0] + else: + def is_us(ep): + return ep.application.name == self.name + local_ep, remote_ep = sorted(rel.endpoints, key=is_us) + if endpoint_name is not None and endpoint_name != local_ep.name: + continue + apps[remote_ep.application.name] = remote_ep.application + return apps + @property def status(self): """Get the application status, as set by the charm's leader. diff --git a/modules/libjuju/juju/client/_client.py b/modules/libjuju/juju/client/_client.py index 2ef0ffd..d959a56 100644 --- a/modules/libjuju/juju/client/_client.py +++ b/modules/libjuju/juju/client/_client.py @@ -1,10 +1,8 @@ # DO NOT CHANGE THIS FILE! This file is auto-generated by facade.py. # Changes will be overwritten/lost when the file is regenerated. -from juju.client._definitions import * # noqa - from juju.client import _client1, _client2, _client3, _client4, _client5 - +from juju.client._definitions import * # noqa CLIENTS = { "1": _client1, @@ -43,7 +41,13 @@ class TypeFactory: @param connection: initialized Connection object. """ - version = connection.facades[cls.__name__[:-6]] + facade_name = cls.__name__ + if not facade_name.endswith('Facade'): + raise TypeError('Unexpected class name: {}'.format(facade_name)) + facade_name = facade_name[:-len('Facade')] + version = connection.facades.get(facade_name) + if version is None: + raise Exception('No facade {} in facades {}'.format(facade_name, connection.facades)) c = lookup_facade(cls.__name__, version) c = c() diff --git a/modules/libjuju/juju/client/_client1.py b/modules/libjuju/juju/client/_client1.py index 3774056..e161973 100644 --- a/modules/libjuju/juju/client/_client1.py +++ b/modules/libjuju/juju/client/_client1.py @@ -1,8 +1,8 @@ # DO NOT CHANGE THIS FILE! This file is auto-generated by facade.py. # Changes will be overwritten/lost when the file is regenerated. -from juju.client.facade import Type, ReturnMapping from juju.client._definitions import * +from juju.client.facade import ReturnMapping, Type class AgentToolsFacade(Type): @@ -6641,5 +6641,3 @@ class UserManagerFacade(Type): _params['include-disabled'] = include_disabled reply = await self.rpc(msg) return reply - - diff --git a/modules/libjuju/juju/client/_client2.py b/modules/libjuju/juju/client/_client2.py index 283e803..6f92a86 100644 --- a/modules/libjuju/juju/client/_client2.py +++ b/modules/libjuju/juju/client/_client2.py @@ -1,8 +1,8 @@ # DO NOT CHANGE THIS FILE! This file is auto-generated by facade.py. # Changes will be overwritten/lost when the file is regenerated. -from juju.client.facade import Type, ReturnMapping from juju.client._definitions import * +from juju.client.facade import ReturnMapping, Type class ActionFacade(Type): @@ -4223,5 +4223,3 @@ class VolumeAttachmentsWatcherFacade(Type): reply = await self.rpc(msg) return reply - - diff --git a/modules/libjuju/juju/client/_client3.py b/modules/libjuju/juju/client/_client3.py index 3f9ef55..b5f4b9d 100644 --- a/modules/libjuju/juju/client/_client3.py +++ b/modules/libjuju/juju/client/_client3.py @@ -1,8 +1,8 @@ # DO NOT CHANGE THIS FILE! This file is auto-generated by facade.py. # Changes will be overwritten/lost when the file is regenerated. -from juju.client.facade import Type, ReturnMapping from juju.client._definitions import * +from juju.client.facade import ReturnMapping, Type class ApplicationFacade(Type): @@ -5216,5 +5216,3 @@ class StorageProvisionerFacade(Type): _params['entities'] = entities reply = await self.rpc(msg) return reply - - diff --git a/modules/libjuju/juju/client/_client4.py b/modules/libjuju/juju/client/_client4.py index 68ee3f9..9c47561 100644 --- a/modules/libjuju/juju/client/_client4.py +++ b/modules/libjuju/juju/client/_client4.py @@ -1,8 +1,8 @@ # DO NOT CHANGE THIS FILE! This file is auto-generated by facade.py. # Changes will be overwritten/lost when the file is regenerated. -from juju.client.facade import Type, ReturnMapping from juju.client._definitions import * +from juju.client.facade import ReturnMapping, Type class ApplicationFacade(Type): @@ -2667,5 +2667,3 @@ class UniterFacade(Type): _params['entities'] = entities reply = await self.rpc(msg) return reply - - diff --git a/modules/libjuju/juju/client/_client5.py b/modules/libjuju/juju/client/_client5.py index 22805ed..f0f1282 100644 --- a/modules/libjuju/juju/client/_client5.py +++ b/modules/libjuju/juju/client/_client5.py @@ -1,8 +1,8 @@ # DO NOT CHANGE THIS FILE! This file is auto-generated by facade.py. # Changes will be overwritten/lost when the file is regenerated. -from juju.client.facade import Type, ReturnMapping from juju.client._definitions import * +from juju.client.facade import ReturnMapping, Type class ApplicationFacade(Type): @@ -2650,5 +2650,3 @@ class UniterFacade(Type): _params['entities'] = entities reply = await self.rpc(msg) return reply - - diff --git a/modules/libjuju/juju/client/_definitions.py b/modules/libjuju/juju/client/_definitions.py index 198784d..fde035f 100644 --- a/modules/libjuju/juju/client/_definitions.py +++ b/modules/libjuju/juju/client/_definitions.py @@ -1,7 +1,7 @@ # DO NOT CHANGE THIS FILE! This file is auto-generated by facade.py. # Changes will be overwritten/lost when the file is regenerated. -from juju.client.facade import Type, ReturnMapping +from juju.client.facade import ReturnMapping, Type class APIHostPortsResult(Type): @@ -8018,5 +8018,3 @@ class ZoneResults(Type): results : typing.Sequence<+T_co>[~ZoneResult]<~ZoneResult> ''' self.results = [ZoneResult.from_json(o) for o in results or []] - - diff --git a/modules/libjuju/juju/client/client.py b/modules/libjuju/juju/client/client.py index 2f3e49d..2721d07 100644 --- a/modules/libjuju/juju/client/client.py +++ b/modules/libjuju/juju/client/client.py @@ -1,8 +1,7 @@ '''Replace auto-generated classes with our own, where necessary. ''' -from . import _client, _definitions, overrides - +from . import _client, _definitions, overrides # isort:skip for o in overrides.__all__: if "Facade" not in o: @@ -31,4 +30,4 @@ for o in overrides.__patches__: if not a.startswith('_'): setattr(c_type, a, getattr(o_type, a)) -from ._client import * # noqa +from ._client import * # noqa, isort:skip diff --git a/modules/libjuju/juju/client/connection.py b/modules/libjuju/juju/client/connection.py index c09468c..bdd1c3f 100644 --- a/modules/libjuju/juju/client/connection.py +++ b/modules/libjuju/juju/client/connection.py @@ -1,28 +1,21 @@ +import asyncio import base64 -import io import json import logging -import os -import random -import shlex import ssl -import string -import subprocess +import urllib.request import weakref -import websockets from concurrent.futures import CancelledError from http.client import HTTPSConnection -from pathlib import Path - -import asyncio -import yaml -from juju import tag, utils +import macaroonbakery.httpbakery as httpbakery +import macaroonbakery.bakery as bakery +import websockets +from juju import errors, tag, utils from juju.client import client -from juju.errors import JujuError, JujuAPIError, JujuConnectionError from juju.utils import IdQueue -log = logging.getLogger("websocket") +log = logging.getLogger('juju.client.connection') class Monitor: @@ -30,7 +23,7 @@ class Monitor: Monitor helper class for our Connection class. Contains a reference to an instantiated Connection, along with a - reference to the Connection.receiver Future. Upon inspecttion of + reference to the Connection.receiver Future. Upon inspection of these objects, this class determines whether the connection is in an 'error', 'connected' or 'disconnected' state. @@ -48,10 +41,6 @@ class Monitor: self.connection = weakref.ref(connection) self.reconnecting = asyncio.Lock(loop=connection.loop) self.close_called = asyncio.Event(loop=connection.loop) - self.receiver_stopped = asyncio.Event(loop=connection.loop) - self.pinger_stopped = asyncio.Event(loop=connection.loop) - self.receiver_stopped.set() - self.pinger_stopped.set() @property def status(self): @@ -81,7 +70,8 @@ class Monitor: return self.DISCONNECTING # connection closed uncleanly (we didn't call connection.close) - if self.receiver_stopped.is_set() or not connection.ws.open: + stopped = connection._receiver_task.stopped.is_set() + if stopped or not connection.ws.open: return self.ERROR # everything is fine! @@ -96,47 +86,91 @@ class Connection: client = await Connection.connect( api_endpoint, model_uuid, username, password, cacert) - # Connect using a controller/model name - client = await Connection.connect_model('local.local:default') - - # Connect to the currently active model - client = await Connection.connect_current() - Note: Any connection method or constructor can accept an optional `loop` argument to override the default event loop from `asyncio.get_event_loop`. """ - DEFAULT_FRAME_SIZE = 'default_frame_size' MAX_FRAME_SIZE = 2**22 "Maximum size for a single frame. Defaults to 4MB." - def __init__( - self, endpoint, uuid, username, password, cacert=None, - macaroons=None, loop=None, max_frame_size=DEFAULT_FRAME_SIZE): - self.endpoint = endpoint - self._endpoint = endpoint + @classmethod + async def connect( + cls, + endpoint=None, + uuid=None, + username=None, + password=None, + cacert=None, + bakery_client=None, + loop=None, + max_frame_size=None, + ): + """Connect to the websocket. + + If uuid is None, the connection will be to the controller. Otherwise it + will be to the model. + :param str endpoint The hostname:port of the controller to connect to. + :param str uuid The model UUID to connect to (None for a + controller-only connection). + :param str username The username for controller-local users (or None + to use macaroon-based login.) + :param str password The password for controller-local users. + :param str cacert The CA certificate of the controller (PEM formatted). + :param httpbakery.Client bakery_client The macaroon bakery client to + to use when performing macaroon-based login. Macaroon tokens + acquired when logging will be saved to bakery_client.cookies. + If this is None, a default bakery_client will be used. + :param loop asyncio.BaseEventLoop The event loop to use for async + operations. + :param max_frame_size The maximum websocket frame size to allow. + """ + self = cls() + if endpoint is None: + raise ValueError('no endpoint provided') self.uuid = uuid - if macaroons: - self.macaroons = macaroons - self.username = '' - self.password = '' - else: - self.macaroons = [] - self.username = username - self.password = password - self.cacert = cacert - self._cacert = cacert + if bakery_client is None: + bakery_client = httpbakery.Client() + self.bakery_client = bakery_client + if username and '@' in username and not username.endswith('@local'): + # We're trying to log in as an external user - we need to use + # macaroon authentication with no username or password. + if password is not None: + raise errors.JujuAuthError('cannot log in as external ' + 'user with a password') + username = None + self.usertag = tag.user(username) + self.password = password self.loop = loop or asyncio.get_event_loop() self.__request_id__ = 0 + + # The following instance variables are initialized by the + # _connect_with_redirect method, but create them here + # as a reminder that they will exist. self.addr = None self.ws = None + self.endpoint = None + self.cacert = None + self.info = None + + # Create that _Task objects but don't start the tasks yet. + self._pinger_task = _Task(self._pinger, self.loop) + self._receiver_task = _Task(self._receiver, self.loop) + self.facades = {} self.messages = IdQueue(loop=self.loop) self.monitor = Monitor(connection=self) - if max_frame_size is self.DEFAULT_FRAME_SIZE: + if max_frame_size is None: max_frame_size = self.MAX_FRAME_SIZE self.max_frame_size = max_frame_size + await self._connect_with_redirect([(endpoint, cacert)]) + return self + + @property + def username(self): + if not self.usertag: + return None + return self.usertag[len('user-'):] @property def is_open(self): @@ -146,39 +180,34 @@ class Connection: return ssl.create_default_context( purpose=ssl.Purpose.CLIENT_AUTH, cadata=cert) - async def open(self): + async def _open(self, endpoint, cacert): if self.uuid: - url = "wss://{}/model/{}/api".format(self.endpoint, self.uuid) + url = "wss://{}/model/{}/api".format(endpoint, self.uuid) else: - url = "wss://{}/api".format(self.endpoint) - - kw = dict() - kw['ssl'] = self._get_ssl(self.cacert) - kw['loop'] = self.loop - kw['max_size'] = self.max_frame_size - self.addr = url - self.ws = await websockets.connect(url, **kw) - self.loop.create_task(self.receiver()) - self.monitor.receiver_stopped.clear() - log.info("Driver connected to juju %s", url) - self.monitor.close_called.clear() - return self + url = "wss://{}/api".format(endpoint) + + return (await websockets.connect( + url, + ssl=self._get_ssl(cacert), + loop=self.loop, + max_size=self.max_frame_size, + ), url, endpoint, cacert) async def close(self): if not self.ws: return self.monitor.close_called.set() - await self.monitor.pinger_stopped.wait() - await self.monitor.receiver_stopped.wait() + await self._pinger_task.stopped.wait() + await self._receiver_task.stopped.wait() await self.ws.close() self.ws = None - async def recv(self, request_id): + async def _recv(self, request_id): if not self.is_open: raise websockets.exceptions.ConnectionClosed(0, 'websocket closed') return await self.messages.get(request_id) - async def receiver(self): + async def _receiver(self): try: while self.is_open: result = await utils.run_with_interrupt( @@ -205,10 +234,8 @@ class Connection: # make pending listeners aware of the error await self.messages.put_all(e) raise - finally: - self.monitor.receiver_stopped.set() - async def pinger(self): + async def _pinger(self): ''' A Controller can time us out if we are silent for too long. This is especially true in JaaS, which has a fairly strict timeout. @@ -232,11 +259,21 @@ class Connection: loop=self.loop) if self.monitor.close_called.is_set(): break - finally: - self.monitor.pinger_stopped.set() - return + except websockets.exceptions.ConnectionClosed: + # The connection has closed - we can't do anything + # more until the connection is restarted. + log.debug('ping failed because of closed connection') + pass async def rpc(self, msg, encoder=None): + '''Make an RPC to the API. The message is encoded as JSON + using the given encoder if any. + :param msg: Parameters for the call (will be encoded as JSON). + :param encoder: Encoder to be used when encoding the message. + :return: The result of the call. + :raises JujuAPIError: When there's an error returned. + :raises JujuError: + ''' self.__request_id__ += 1 msg['request-id'] = self.__request_id__ if'params' not in msg: @@ -244,7 +281,12 @@ class Connection: if "version" not in msg: msg['version'] = self.facades[msg['type']] outgoing = json.dumps(msg, indent=2, cls=encoder) + log.debug('connection {} -> {}'.format(id(self), outgoing)) for attempt in range(3): + if self.monitor.status == Monitor.DISCONNECTED: + # closed cleanly; shouldn't try to reconnect + raise websockets.exceptions.ConnectionClosed( + 0, 'websocket closed') try: await self.ws.send(outgoing) break @@ -257,14 +299,19 @@ class Connection: # be cancelled when the pinger is cancelled by the reconnect, # and we don't want the reconnect to be aborted halfway through await asyncio.wait([self.reconnect()], loop=self.loop) - result = await self.recv(msg['request-id']) + if self.monitor.status != Monitor.CONNECTED: + # reconnect failed; abort and shutdown + log.error('RPC: Automatic reconnect failed') + raise + result = await self._recv(msg['request-id']) + log.debug('connection {} <- {}'.format(id(self), result)) if not result: return result if 'error' in result: # API Error Response - raise JujuAPIError(result) + raise errors.JujuAPIError(result) if 'response' not in result: # This may never happen @@ -272,30 +319,34 @@ class Connection: if 'results' in result['response']: # Check for errors in a result list. - errors = [] + # TODO This loses the results that might have succeeded. + # Perhaps JujuError should return all the results including + # errors, or perhaps a keyword parameter to the rpc method + # could be added to trigger this behaviour. + err_results = [] for res in result['response']['results']: if res.get('error', {}).get('message'): - errors.append(res['error']['message']) - if errors: - raise JujuError(errors) + err_results.append(res['error']['message']) + if err_results: + raise errors.JujuError(err_results) elif result['response'].get('error', {}).get('message'): - raise JujuError(result['response']['error']['message']) + raise errors.JujuError(result['response']['error']['message']) return result - def http_headers(self): + def _http_headers(self): """Return dictionary of http headers necessary for making an http connection to the endpoint of this Connection. :return: Dictionary of headers """ - if not self.username: + if not self.usertag: return {} creds = u'{}:{}'.format( - tag.user(self.username), + self.usertag, self.password or '' ) token = base64.b64encode(creds.encode()) @@ -328,70 +379,46 @@ class Connection: "/model/{}".format(self.uuid) if self.uuid else "" ) - return conn, self.http_headers(), path + return conn, self._http_headers(), path async def clone(self): """Return a new Connection, connected to the same websocket endpoint as this one. """ - return await Connection.connect( - self.endpoint, - self.uuid, - self.username, - self.password, - self.cacert, - self.macaroons, - self.loop, - self.max_frame_size, - ) + return await Connection.connect(**self.connect_params()) + + def connect_params(self): + """Return a tuple of parameters suitable for passing to + Connection.connect that can be used to make a new connection + to the same controller (and model if specified. The first + element in the returned tuple holds the endpoint argument; + the other holds a dict of the keyword args. + """ + return { + 'endpoint': self.endpoint, + 'uuid': self.uuid, + 'username': self.username, + 'password': self.password, + 'cacert': self.cacert, + 'bakery_client': self.bakery_client, + 'loop': self.loop, + 'max_frame_size': self.max_frame_size, + } async def controller(self): """Return a Connection to the controller at self.endpoint - """ return await Connection.connect( self.endpoint, - None, - self.username, - self.password, - self.cacert, - self.macaroons, - self.loop, + username=self.username, + password=self.password, + cacert=self.cacert, + bakery_client=self.bakery_client, + loop=self.loop, + max_frame_size=self.max_frame_size, ) - async def _try_endpoint(self, endpoint, cacert): - success = False - result = None - new_endpoints = [] - - self.endpoint = endpoint - self.cacert = cacert - await self.open() - try: - result = await self.login() - if 'discharge-required-error' in result['response']: - log.info('Macaroon discharge required, disconnecting') - else: - # successful login! - log.info('Authenticated') - success = True - except JujuAPIError as e: - if e.error_code != 'redirection required': - raise - log.info('Controller requested redirect') - redirect_info = await self.redirect_info() - redir_cacert = redirect_info['ca-cert'] - new_endpoints = [ - ("{value}:{port}".format(**s), redir_cacert) - for servers in redirect_info['servers'] - for s in servers if s["scope"] == 'public' - ] - finally: - if not success: - await self.close() - return success, result, new_endpoints - async def reconnect(self): """ Force a reconnection. """ @@ -400,256 +427,149 @@ class Connection: return async with monitor.reconnecting: await self.close() - await self._connect() - - async def _connect(self): - endpoints = [(self._endpoint, self._cacert)] - while endpoints: - _endpoint, _cacert = endpoints.pop(0) - success, result, new_endpoints = await self._try_endpoint( - _endpoint, _cacert) - if success: + await self._connect_with_login([(self.endpoint, self.cacert)]) + + async def _connect(self, endpoints): + if len(endpoints) == 0: + raise errors.JujuConnectionError('no endpoints to connect to') + + async def _try_endpoint(endpoint, cacert, delay): + if delay: + await asyncio.sleep(delay) + return await self._open(endpoint, cacert) + + # Try all endpoints in parallel, with slight increasing delay (+100ms + # for each subsequent endpoint); the delay allows us to prefer the + # earlier endpoints over the latter. Use first successful connection. + tasks = [self.loop.create_task(_try_endpoint(endpoint, cacert, + 0.1 * i)) + for i, (endpoint, cacert) in enumerate(endpoints)] + for task in asyncio.as_completed(tasks, loop=self.loop): + try: + result = await task break - endpoints.extend(new_endpoints) + except ConnectionError: + continue # ignore; try another endpoint else: - # ran out of endpoints without a successful login - raise JujuConnectionError("Couldn't authenticate to {}".format( - self._endpoint)) - - response = result['response'] - self.info = response.copy() - self.build_facades(response.get('facades', {})) - self.loop.create_task(self.pinger()) - self.monitor.pinger_stopped.clear() + raise errors.JujuConnectionError( + 'Unable to connect to any endpoint: {}'.format(', '.join([ + endpoint for endpoint, cacert in endpoints]))) + for task in tasks: + task.cancel() + self.ws = result[0] + self.addr = result[1] + self.endpoint = result[2] + self.cacert = result[3] + self._receiver_task.start() + log.info("Driver connected to juju %s", self.addr) + self.monitor.close_called.clear() - @classmethod - async def connect( - cls, endpoint, uuid, username, password, cacert=None, - macaroons=None, loop=None, max_frame_size=None): + async def _connect_with_login(self, endpoints): """Connect to the websocket. If uuid is None, the connection will be to the controller. Otherwise it will be to the model. - - """ - client = cls(endpoint, uuid, username, password, cacert, macaroons, - loop, max_frame_size) - await client._connect() - return client - - @classmethod - async def connect_current(cls, loop=None, max_frame_size=None): - """Connect to the currently active model. - - """ - jujudata = JujuData() - - controller_name = jujudata.current_controller() - if not controller_name: - raise JujuConnectionError('No current controller') - - model_name = jujudata.current_model() - - return await cls.connect_model( - '{}:{}'.format(controller_name, model_name), loop, max_frame_size) - - @classmethod - async def connect_current_controller(cls, loop=None, max_frame_size=None): - """Connect to the currently active controller. - + :return: The response field of login response JSON object. """ - jujudata = JujuData() - controller_name = jujudata.current_controller() - if not controller_name: - raise JujuConnectionError('No current controller') - - return await cls.connect_controller(controller_name, loop, - max_frame_size) - - @classmethod - async def connect_controller(cls, controller_name, loop=None, - max_frame_size=None): - """Connect to a controller by name. - - """ - jujudata = JujuData() - controller = jujudata.controllers()[controller_name] - endpoint = controller['api-endpoints'][0] - cacert = controller.get('ca-cert') - accounts = jujudata.accounts()[controller_name] - username = accounts['user'] - password = accounts.get('password') - macaroons = get_macaroons(controller_name) if not password else None - - return await cls.connect( - endpoint, None, username, password, cacert, macaroons, loop, - max_frame_size) - - @classmethod - async def connect_model(cls, model, loop=None, max_frame_size=None): - """Connect to a model by name. - - :param str model: [:] + success = False + try: + await self._connect(endpoints) + # It's possible that we may get several discharge-required errors, + # corresponding to different levels of authentication, so retry + # a few times. + for i in range(0, 2): + result = (await self.login())['response'] + macaroonJSON = result.get('discharge-required') + if macaroonJSON is None: + self.info = result + success = True + return result + macaroon = bakery.Macaroon.from_dict(macaroonJSON) + self.bakery_client.handle_error( + httpbakery.Error( + code=httpbakery.ERR_DISCHARGE_REQUIRED, + message=result.get('discharge-required-error'), + version=macaroon.version, + info=httpbakery.ErrorInfo( + macaroon=macaroon, + macaroon_path=result.get('macaroon-path'), + ), + ), + # note: remove the port number. + 'https://' + self.endpoint + '/', + ) + raise errors.JujuAuthError('failed to authenticate ' + 'after several attempts') + finally: + if not success: + await self.close() - """ - jujudata = JujuData() + async def _connect_with_redirect(self, endpoints): + try: + login_result = await self._connect_with_login(endpoints) + except errors.JujuRedirectException as e: + login_result = await self._connect_with_login(e.endpoints) + self._build_facades(login_result.get('facades', {})) + self._pinger_task.start() - if ':' in model: - # explicit controller given - controller_name, model_name = model.split(':') - else: - # use the current controller if one isn't explicitly given - controller_name = jujudata.current_controller() - model_name = model - - accounts = jujudata.accounts()[controller_name] - username = accounts['user'] - # model name must include a user prefix, so add it if it doesn't - if '/' not in model_name: - model_name = '{}/{}'.format(username, model_name) - - controller = jujudata.controllers()[controller_name] - endpoint = controller['api-endpoints'][0] - cacert = controller.get('ca-cert') - password = accounts.get('password') - models = jujudata.models()[controller_name] - model_uuid = models['models'][model_name]['uuid'] - macaroons = get_macaroons(controller_name) if not password else None - - return await cls.connect( - endpoint, model_uuid, username, password, cacert, macaroons, loop, - max_frame_size) - - def build_facades(self, facades): + def _build_facades(self, facades): self.facades.clear() for facade in facades: self.facades[facade['name']] = facade['versions'][-1] async def login(self): - username = self.username - if username and not username.startswith('user-'): - username = 'user-{}'.format(username) - - result = await self.rpc({ - "type": "Admin", - "request": "Login", - "version": 3, - "params": { - "auth-tag": username, - "credentials": self.password, - "nonce": "".join(random.sample(string.printable, 12)), - "macaroons": self.macaroons - }}) - return result + params = {} + if self.password: + params['auth-tag'] = self.usertag + params['credentials'] = self.password + else: + macaroons = _macaroons_for_domain(self.bakery_client.cookies, + self.endpoint) + params['macaroons'] = [[bakery.macaroon_to_dict(m) for m in ms] + for ms in macaroons] - async def redirect_info(self): try: - result = await self.rpc({ + return await self.rpc({ "type": "Admin", - "request": "RedirectInfo", + "request": "Login", "version": 3, + "params": params, }) - except JujuAPIError as e: - if e.message == 'not redirected': - return None - raise - return result['response'] - - -class JujuData: - def __init__(self): - self.path = os.environ.get('JUJU_DATA') or '~/.local/share/juju' - self.path = os.path.abspath(os.path.expanduser(self.path)) - - def current_controller(self): - cmd = shlex.split('juju list-controllers --format yaml') - output = subprocess.check_output(cmd) - output = yaml.safe_load(output) - return output.get('current-controller', '') - - def current_model(self, controller_name=None): - if not controller_name: - controller_name = self.current_controller() - models = self.models()[controller_name] - if 'current-model' not in models: - raise JujuError('No current model') - return models['current-model'] - - def controllers(self): - return self._load_yaml('controllers.yaml', 'controllers') - - def models(self): - return self._load_yaml('models.yaml', 'controllers') - - def accounts(self): - return self._load_yaml('accounts.yaml', 'controllers') - - def credentials(self): - return self._load_yaml('credentials.yaml', 'credentials') + except errors.JujuAPIError as e: + if e.error_code != 'redirection required': + raise + log.info('Controller requested redirect') + # Fetch additional redirection information now so that + # we can safely close the connection after login + # fails. + redirect_info = (await self.rpc({ + "type": "Admin", + "request": "RedirectInfo", + "version": 3, + }))['response'] + raise errors.JujuRedirectException(redirect_info) from e - def load_credential(self, cloud, name=None): - """Load a local credential. - :param str cloud: Name of cloud to load credentials from. - :param str name: Name of credential. If None, the default credential - will be used, if available. - :returns: A CloudCredential instance, or None. - """ - try: - cloud = tag.untag('cloud-', cloud) - creds_data = self.credentials()[cloud] - if not name: - default_credential = creds_data.pop('default-credential', None) - default_region = creds_data.pop('default-region', None) # noqa - if default_credential: - name = creds_data['default-credential'] - elif len(creds_data) == 1: - name = list(creds_data)[0] - else: - return None, None - cred_data = creds_data[name] - auth_type = cred_data.pop('auth-type') - return name, client.CloudCredential( - auth_type=auth_type, - attrs=cred_data, - ) - except (KeyError, FileNotFoundError): - return None, None - - def _load_yaml(self, filename, key): - filepath = os.path.join(self.path, filename) - with io.open(filepath, 'rt') as f: - return yaml.safe_load(f)[key] - - -def get_macaroons(controller_name=None): - """Decode and return macaroons from default ~/.go-cookies +class _Task: + def __init__(self, task, loop): + self.stopped = asyncio.Event(loop=loop) + self.stopped.set() + self.task = task + self.loop = loop - """ - cookie_files = [] - if controller_name: - cookie_files.append('~/.local/share/juju/cookies/{}.json'.format( - controller_name)) - cookie_files.append('~/.go-cookies') - for cookie_file in cookie_files: - cookie_file = Path(cookie_file).expanduser() - if cookie_file.exists(): + def start(self): + async def run(): try: - cookies = json.loads(cookie_file.read_text()) - break - except (OSError, ValueError): - log.warn("Couldn't load macaroons from %s", cookie_file) - return [] - else: - log.warn("Couldn't load macaroons from %s", ' or '.join(cookie_files)) - return [] - - base64_macaroons = [ - c['Value'] for c in cookies - if c['Name'].startswith('macaroon-') and c['Value'] - ] - - return [ - json.loads(base64.b64decode(value).decode('utf-8')) - for value in base64_macaroons - ] + return await(self.task()) + finally: + self.stopped.set() + self.stopped.clear() + self.loop.create_task(run()) + + +def _macaroons_for_domain(cookies, domain): + '''Return any macaroons from the given cookie jar that + apply to the given domain name.''' + req = urllib.request.Request('https://' + domain + '/') + cookies.add_cookie_header(req) + return httpbakery.extract_macaroons(req) diff --git a/modules/libjuju/juju/client/connector.py b/modules/libjuju/juju/client/connector.py new file mode 100644 index 0000000..64fbe44 --- /dev/null +++ b/modules/libjuju/juju/client/connector.py @@ -0,0 +1,147 @@ +import asyncio +import logging +import copy + +import macaroonbakery.httpbakery as httpbakery +from juju.client.connection import Connection +from juju.client.jujudata import FileJujuData +from juju.errors import JujuConnectionError, JujuError + +log = logging.getLogger('connector') + + +class NoConnectionException(Exception): + '''Raised by Connector when the connection method is called + and there is no current connection.''' + pass + + +class Connector: + '''This class abstracts out a reconnectable client that can connect + to controllers and models found in the Juju data files. + ''' + def __init__( + self, + loop=None, + max_frame_size=None, + bakery_client=None, + jujudata=None, + ): + '''Initialize a connector that will use the given parameters + by default when making a new connection''' + self.max_frame_size = max_frame_size + self.loop = loop or asyncio.get_event_loop() + self.bakery_client = bakery_client + self._connection = None + self.controller_name = None + self.model_name = None + self.jujudata = jujudata or FileJujuData() + + def is_connected(self): + '''Report whether there is a currently connected controller or not''' + return self._connection is not None + + def connection(self): + '''Return the current connection; raises an exception if there + is no current connection.''' + if not self.is_connected(): + raise NoConnectionException('not connected') + return self._connection + + async def connect(self, **kwargs): + """Connect to an arbitrary Juju model. + + kwargs are passed through to Connection.connect() + """ + kwargs.setdefault('loop', self.loop) + kwargs.setdefault('max_frame_size', self.max_frame_size) + kwargs.setdefault('bakery_client', self.bakery_client) + self._connection = await Connection.connect(**kwargs) + + async def disconnect(self): + """Shut down the watcher task and close websockets. + """ + if self._connection: + log.debug('Closing model connection') + await self._connection.close() + self._connection = None + + async def connect_controller(self, controller_name=None): + """Connect to a controller by name. If the name is empty, it + connect to the current controller. + """ + if not controller_name: + controller_name = self.jujudata.current_controller() + if not controller_name: + raise JujuConnectionError('No current controller') + + controller = self.jujudata.controllers()[controller_name] + # TODO change Connection so we can pass all the endpoints + # instead of just the first. + endpoint = controller['api-endpoints'][0] + accounts = self.jujudata.accounts().get(controller_name, {}) + + await self.connect( + endpoint=endpoint, + uuid=None, + username=accounts.get('user'), + password=accounts.get('password'), + cacert=controller.get('ca-cert'), + bakery_client=self.bakery_client_for_controller(controller_name), + ) + self.controller_name = controller_name + + async def connect_model(self, model_name=None): + """Connect to a model by name. If either controller or model + parts of the name are empty, the current controller and/or model + will be used. + + :param str model: : + """ + + try: + controller_name, model_name = self.jujudata.parse_model(model_name) + controller = self.jujudata.controllers().get(controller_name) + except JujuError as e: + raise JujuConnectionError(e.message) from e + if controller is None: + raise JujuConnectionError('Controller {} not found'.format( + controller_name)) + # TODO change Connection so we can pass all the endpoints + # instead of just the first one. + endpoint = controller['api-endpoints'][0] + account = self.jujudata.accounts().get(controller_name, {}) + models = self.jujudata.models().get(controller_name, {}).get('models', + {}) + if model_name not in models: + raise JujuConnectionError('Model not found: {}'.format(model_name)) + + # TODO if there's no record for the required model name, connect + # to the controller to find out the model's uuid, then connect + # to that. This will let connect_model work with models that + # haven't necessarily synced with the local juju data, + # and also remove the need for base.CleanModel to + # subclass JujuData. + await self.connect( + endpoint=endpoint, + uuid=models[model_name]['uuid'], + username=account.get('user'), + password=account.get('password'), + cacert=controller.get('ca-cert'), + bakery_client=self.bakery_client_for_controller(controller_name), + ) + self.controller_name = controller_name + self.model_name = controller_name + ':' + model_name + + def bakery_client_for_controller(self, controller_name): + '''Make a copy of the bakery client with a the appropriate controller's + cookiejar in it. + ''' + bakery_client = self.bakery_client + if bakery_client: + bakery_client = copy.copy(bakery_client) + else: + bakery_client = httpbakery.Client() + bakery_client.cookies = self.jujudata.cookies_for_controller( + controller_name) + return bakery_client diff --git a/modules/libjuju/juju/client/facade.py b/modules/libjuju/juju/client/facade.py index c015c5f..1c7baa0 100644 --- a/modules/libjuju/juju/client/facade.py +++ b/modules/libjuju/juju/client/facade.py @@ -1,16 +1,16 @@ import argparse import builtins -from collections import defaultdict import functools -from glob import glob import json import keyword -from pathlib import Path import pprint import re import textwrap -from typing import Sequence, Mapping, TypeVar, Any, Union import typing +from collections import defaultdict +from glob import glob +from pathlib import Path +from typing import Any, Mapping, Sequence, TypeVar, Union from . import codegen diff --git a/modules/libjuju/juju/client/gocookies.py b/modules/libjuju/juju/client/gocookies.py new file mode 100644 index 0000000..a8a0df8 --- /dev/null +++ b/modules/libjuju/juju/client/gocookies.py @@ -0,0 +1,102 @@ +import datetime +import http.cookiejar as cookiejar +import json +import time + +import pyrfc3339 + + +class GoCookieJar(cookiejar.FileCookieJar): + '''A CookieJar implementation that reads and writes cookies + to the cookiejar format as understood by the Go package + github.com/juju/persistent-cookiejar.''' + def _really_load(self, f, filename, ignore_discard, ignore_expires): + '''Implement the _really_load method called by FileCookieJar + to implement the actual cookie loading''' + data = json.load(f) or [] + now = time.time() + for cookie in map(_new_py_cookie, data): + if not ignore_expires and cookie.is_expired(now): + continue + self.set_cookie(cookie) + + def save(self, filename=None, ignore_discard=False, ignore_expires=False): + '''Implement the FileCookieJar abstract method.''' + if filename is None: + if self.filename is not None: + filename = self.filename + else: + raise ValueError(cookiejar.MISSING_FILENAME_TEXT) + + # TODO: obtain file lock, read contents of file, and merge with + # current content. + go_cookies = [] + now = time.time() + for cookie in self: + if not ignore_discard and cookie.discard: + continue + if not ignore_expires and cookie.is_expired(now): + continue + go_cookies.append(_new_go_cookie(cookie)) + with open(filename, "w") as f: + f.write(json.dumps(go_cookies)) + + +def _new_py_cookie(go_cookie): + '''Convert a Go-style JSON-unmarshaled cookie into a Python cookie''' + expires = None + if go_cookie.get('Expires') is not None: + t = pyrfc3339.parse(go_cookie['Expires']) + expires = t.timestamp() + return cookiejar.Cookie( + version=0, + name=go_cookie['Name'], + value=go_cookie['Value'], + port=None, + port_specified=False, + # Unfortunately Python cookies don't record the original + # host that the cookie came from, so we'll just use Domain + # for that purpose, and record that the domain was specified, + # even though it probably was not. This means that + # we won't correctly record the CanonicalHost entry + # when writing the cookie file after reading it. + domain=go_cookie['Domain'], + domain_specified=not go_cookie['HostOnly'], + domain_initial_dot=False, + path=go_cookie['Path'], + path_specified=True, + secure=go_cookie['Secure'], + expires=expires, + discard=False, + comment=None, + comment_url=None, + rest=None, + rfc2109=False, + ) + + +def _new_go_cookie(py_cookie): + '''Convert a python cookie to the JSON-marshalable Go-style cookie form.''' + # TODO (perhaps): + # HttpOnly + # Creation + # LastAccess + # Updated + # not done properly: CanonicalHost. + go_cookie = { + 'Name': py_cookie.name, + 'Value': py_cookie.value, + 'Domain': py_cookie.domain, + 'HostOnly': not py_cookie.domain_specified, + 'Persistent': not py_cookie.discard, + 'Secure': py_cookie.secure, + 'CanonicalHost': py_cookie.domain, + } + if py_cookie.path_specified: + go_cookie['Path'] = py_cookie.path + if py_cookie.expires is not None: + unix_time = datetime.datetime.fromtimestamp(py_cookie.expires) + # Note: fromtimestamp bizarrely produces a time without + # a time zone, so we need to use accept_naive. + go_cookie['Expires'] = pyrfc3339.generate(unix_time, accept_naive=True) + return go_cookie diff --git a/modules/libjuju/juju/client/jujudata.py b/modules/libjuju/juju/client/jujudata.py new file mode 100644 index 0000000..8b844c2 --- /dev/null +++ b/modules/libjuju/juju/client/jujudata.py @@ -0,0 +1,219 @@ +import abc +import io +import os +import pathlib + +import juju.client.client as jujuclient +import yaml +from juju import tag +from juju.client.gocookies import GoCookieJar +from juju.errors import JujuError + + +class NoModelException(Exception): + pass + + +class JujuData: + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def current_controller(self): + '''Return the current controller name''' + raise NotImplementedError() + + @abc.abstractmethod + def controllers(self): + '''Return all the currently known controllers as a dict + mapping controller name to a dict containing the + following string keys: + uuid: The UUID of the controller + api-endpoints: A list of host:port addresses for the controller. + ca-cert: the PEM-encoded CA cert of the controller (optional) + + This is compatible with the "controllers" entry in the YAML-unmarshaled data + stored in ~/.local/share/juju/controllers.yaml. + ''' + raise NotImplementedError() + + @abc.abstractmethod + def models(self): + '''Return all the currently known models as a dict + containing a key for each known controller, + each holding a dict value containing an optional "current-model" + key (the name of the current model for that controller, + if there is one), and a dict mapping fully-qualified + model names to a dict containing a "uuid" key with the + key for that model. + This is compatible with the YAML-unmarshaled data + stored in ~/.local/share/juju/models.yaml. + ''' + raise NotImplementedError() + + @abc.abstractmethod + def accounts(self): + '''Return the currently known accounts, as a dict + containing a key for each known controller, with + each value holding a dict with the following keys: + + user: The username to use when logging into the controller (str) + password: The password to use when logging into the controller (str, optional) + ''' + raise NotImplementedError() + + @abc.abstractmethod + def cookies_for_controller(self, controller_name): + '''Return the cookie jar to use when connecting to the + controller with the given name. + :return http.cookiejar.CookieJar + ''' + raise NotImplementedError() + + @abc.abstractmethod + def current_model(self, controller_name=None, model_only=False): + '''Return the current model, qualified by its controller name. + If controller_name is specified, the current model for + that controller will be returned. + If model_only is true, only the model name, not qualified by + its controller name, will be returned. + ''' + raise NotImplementedError() + + def parse_model(self, model): + """Split the given model_name into controller and model parts. + If the controller part is empty, the current controller will be used. + If the model part is empty, the current model will be used for + the controller. + The returned model name will always be qualified with a username. + :param model str: The model name to parse. + :return (str, str): The controller and model names. + """ + # TODO if model is empty, use $JUJU_MODEL environment variable. + if model and ':' in model: + # explicit controller given + controller_name, model_name = model.split(':') + else: + # use the current controller if one isn't explicitly given + controller_name = self.current_controller() + model_name = model + if not controller_name: + controller_name = self.current_controller() + if not model_name: + model_name = self.current_model(controller_name, model_only=True) + if not model_name: + raise NoModelException('no current model') + + if '/' not in model_name: + # model name doesn't include a user prefix, so add one + # by using the current user for the controller. + accounts = self.accounts().get(controller_name) + if accounts is None: + raise JujuError('No account found for controller {} '.format(controller_name)) + username = accounts.get('user') + if username is None: + raise JujuError('No username found for controller {}'.format(controller_name)) + model_name = username + "/" + model_name + + return controller_name, model_name + + +class FileJujuData(JujuData): + '''Provide access to the Juju client configuration files. + Any configuration file is read once and then cached.''' + def __init__(self): + self.path = os.environ.get('JUJU_DATA') or '~/.local/share/juju' + self.path = os.path.abspath(os.path.expanduser(self.path)) + # _loaded keeps track of the loaded YAML from + # the Juju data files so we don't need to load the same + # file many times. + self._loaded = {} + + def refresh(self): + '''Forget the cache of configuration file data''' + self._loaded = {} + + def current_controller(self): + '''Return the current controller name''' + return self._load_yaml('controllers.yaml', 'current-controller') + + def current_model(self, controller_name=None, model_only=False): + '''Return the current model, qualified by its controller name. + If controller_name is specified, the current model for + that controller will be returned. + + If model_only is true, only the model name, not qualified by + its controller name, will be returned. + ''' + # TODO respect JUJU_MODEL environment variable. + if not controller_name: + controller_name = self.current_controller() + if not controller_name: + raise JujuError('No current controller') + models = self.models()[controller_name] + if 'current-model' not in models: + return None + if model_only: + return models['current-model'] + return controller_name + ':' + models['current-model'] + + def load_credential(self, cloud, name=None): + """Load a local credential. + + :param str cloud: Name of cloud to load credentials from. + :param str name: Name of credential. If None, the default credential + will be used, if available. + :return: A CloudCredential instance, or None. + """ + try: + cloud = tag.untag('cloud-', cloud) + creds_data = self.credentials()[cloud] + if not name: + default_credential = creds_data.pop('default-credential', None) + default_region = creds_data.pop('default-region', None) # noqa + if default_credential: + name = creds_data['default-credential'] + elif len(creds_data) == 1: + name = list(creds_data)[0] + else: + return None, None + cred_data = creds_data[name] + auth_type = cred_data.pop('auth-type') + return name, jujuclient.CloudCredential( + auth_type=auth_type, + attrs=cred_data, + ) + except (KeyError, FileNotFoundError): + return None, None + + def controllers(self): + return self._load_yaml('controllers.yaml', 'controllers') + + def models(self): + return self._load_yaml('models.yaml', 'controllers') + + def accounts(self): + return self._load_yaml('accounts.yaml', 'controllers') + + def credentials(self): + return self._load_yaml('credentials.yaml', 'credentials') + + def _load_yaml(self, filename, key): + if filename in self._loaded: + # Data already exists in the cache. + return self._loaded[filename].get(key) + # TODO use the file lock like Juju does. + filepath = os.path.join(self.path, filename) + with io.open(filepath, 'rt') as f: + data = yaml.safe_load(f) + self._loaded[filename] = data + return data.get(key) + + def cookies_for_controller(self, controller_name): + f = pathlib.Path(self.path) / 'cookies' / (controller_name + '.json') + if not f.exists(): + f = pathlib.Path('~/.go-cookies').expanduser() + # TODO if neither cookie file exists, where should + # we create the cookies? + jar = GoCookieJar(str(f)) + jar.load() + return jar diff --git a/modules/libjuju/juju/client/overrides.py b/modules/libjuju/juju/client/overrides.py index 5e98e56..8b29de7 100644 --- a/modules/libjuju/juju/client/overrides.py +++ b/modules/libjuju/juju/client/overrides.py @@ -1,10 +1,8 @@ -from collections import namedtuple import re +from collections import namedtuple +from . import _client, _definitions from .facade import ReturnMapping, Type, TypeEncoder -from .import _client -from .import _definitions - __all__ = [ 'Delta', diff --git a/modules/libjuju/juju/controller.py b/modules/libjuju/juju/controller.py index 55ea55e..957ab85 100644 --- a/modules/libjuju/juju/controller.py +++ b/modules/libjuju/juju/controller.py @@ -1,69 +1,101 @@ import asyncio import logging -from . import errors -from . import tag -from . import utils -from .client import client -from .client import connection -from .model import Model +from . import errors, tag, utils +from .client import client, connector from .user import User log = logging.getLogger(__name__) -class Controller(object): - def __init__(self, loop=None, - max_frame_size=connection.Connection.DEFAULT_FRAME_SIZE): +class Controller: + def __init__( + self, + loop=None, + max_frame_size=None, + bakery_client=None, + jujudata=None, + ): """Instantiate a new Controller. One of the connect_* methods will need to be called before this object can be used for anything interesting. - :param loop: an asyncio event loop + If jujudata is None, jujudata.FileJujuData will be used. + :param loop: an asyncio event loop + :param max_frame_size: See + `juju.client.connection.Connection.MAX_FRAME_SIZE` + :param bakery_client httpbakery.Client: The bakery client to use + for macaroon authorization. + :param jujudata JujuData: The source for current controller information. """ - self.loop = loop or asyncio.get_event_loop() - self.max_frame_size = None - self.connection = None - self.controller_name = None + self._connector = connector.Connector( + loop=loop, + max_frame_size=max_frame_size, + bakery_client=bakery_client, + jujudata=jujudata, + ) - async def connect( - self, endpoint, username, password, cacert=None, macaroons=None): - """Connect to an arbitrary Juju controller. + async def __aenter__(self): + await self.connect() + return self - """ - self.connection = await connection.Connection.connect( - endpoint, None, username, password, cacert, macaroons, - max_frame_size=self.max_frame_size) + async def __aexit__(self, exc_type, exc, tb): + await self.disconnect() - async def connect_current(self): - """Connect to the current Juju controller. + @property + def loop(self): + return self._connector.loop - """ - jujudata = connection.JujuData() - controller_name = jujudata.current_controller() - if not controller_name: - raise errors.JujuConnectionError('No current controller') - return await self.connect_controller(controller_name) + async def connect(self, controller_name=None, **kwargs): + """Connect to a Juju controller. + + If any arguments are specified other than controller_name, + then controller_name must be None and an explicit + connection will be made using Connection.connect + using those parameters (the 'uuid' parameter must + be absent or None). - async def connect_controller(self, controller_name): - """Connect to a Juju controller by name. + Otherwise, if controller_name is None, connect to the + current controller. + Otherwise, controller_name must specify the name + of a known controller. """ - self.connection = ( - await connection.Connection.connect_controller( - controller_name, max_frame_size=self.max_frame_size)) - self.controller_name = controller_name + await self.disconnect() + if not kwargs: + await self._connector.connect_controller(controller_name) + else: + if controller_name is not None: + raise ValueError('controller name may not be specified with other connect parameters') + if kwargs.get('uuid') is not None: + # A UUID implies a model connection, not a controller connection. + raise ValueError('model UUID specified when connecting to controller') + await self._connector.connect(**kwargs) + + async def _connect_direct(self, **kwargs): + await self.disconnect() + await self._connector.connect(**kwargs) + + def is_connected(self): + """Reports whether the Controller is currently connected.""" + return self._connector.is_connected() + + def connection(self): + """Return the current Connection object. It raises an exception + if the Controller is disconnected""" + return self._connector.connection() + + @property + def controller_name(self): + return self._connector.controller_name async def disconnect(self): """Shut down the watcher task and close websockets. """ - if self.connection and self.connection.is_open: - log.debug('Closing controller connection') - await self.connection.close() - self.connection = None + await self._connector.disconnect() async def add_credential(self, name=None, credential=None, cloud=None, owner=None): @@ -84,20 +116,19 @@ class Controller(object): cloud = await self.get_cloud() if not owner: - owner = self.connection.info['user-info']['identity'] + owner = self.connection().info['user-info']['identity'] if credential and not name: raise errors.JujuError('Name must be provided for credential') if not credential: - name, credential = connection.JujuData().load_credential(cloud, - name) + name, credential = self._connector.jujudata.load_credential(cloud, name) if credential is None: - raise errors.JujuError('Unable to find credential: ' - '{}'.format(name)) + raise errors.JujuError( + 'Unable to find credential: {}'.format(name)) log.debug('Uploading credential %s', name) - cloud_facade = client.CloudFacade.from_connection(self.connection) + cloud_facade = client.CloudFacade.from_connection(self.connection()) await cloud_facade.UpdateCredentials([ client.UpdateCloudCredential( tag=tag.credential(cloud, tag.untag('user-', owner), name), @@ -121,12 +152,12 @@ class Controller(object): the current user. :param dict config: Model configuration. :param str region: Region in which to create the model. - + :return Model: A connection to the newly created model. """ model_facade = client.ModelManagerFacade.from_connection( - self.connection) + self.connection()) - owner = owner or self.connection.info['user-info']['identity'] + owner = owner or self.connection().info['user-info']['identity'] cloud_name = cloud_name or await self.get_cloud() try: @@ -153,7 +184,7 @@ class Controller(object): if not config or 'authorized-keys' not in config: config = config or {} config['authorized-keys'] = await utils.read_ssh_key( - loop=self.loop) + loop=self._connector.loop) model_info = await model_facade.CreateModel( tag.cloud(cloud_name), @@ -163,17 +194,11 @@ class Controller(object): owner, region ) - - model = Model() - await model.connect( - self.connection.endpoint, - model_info.uuid, - self.connection.username, - self.connection.password, - self.connection.cacert, - self.connection.macaroons, - loop=self.loop, - ) + from juju.model import Model + model = Model(jujudata=self._connector.jujudata) + kwargs = self.connection().connect_params() + kwargs['uuid'] = model_info.uuid + await model._connect_direct(**kwargs) return model @@ -183,12 +208,12 @@ class Controller(object): :param str \*models: Names or UUIDs of models to destroy """ - uuids = await self._model_uuids() + uuids = await self.model_uuids() models = [uuids[model] if model in uuids else model for model in models] model_facade = client.ModelManagerFacade.from_connection( - self.connection) + self.connection()) log.debug( 'Destroying model%s %s', @@ -212,7 +237,7 @@ class Controller(object): """ if not display_name: display_name = username - user_facade = client.UserManagerFacade.from_connection(self.connection) + user_facade = client.UserManagerFacade.from_connection(self.connection()) users = [client.AddUser(display_name=display_name, username=username, password=password)] @@ -223,7 +248,7 @@ class Controller(object): """Remove a user from this controller. """ client_facade = client.UserManagerFacade.from_connection( - self.connection) + self.connection()) user = tag.user(username) await client_facade.RemoveUser([client.Entity(user)]) @@ -234,7 +259,7 @@ class Controller(object): :param str password: New password """ - user_facade = client.UserManagerFacade.from_connection(self.connection) + user_facade = client.UserManagerFacade.from_connection(self.connection()) entity = client.EntityPassword(password, tag.user(username)) return await user_facade.SetPassword([entity]) @@ -246,7 +271,7 @@ class Controller(object): """ controller_facade = client.ControllerFacade.from_connection( - self.connection) + self.connection()) return await controller_facade.DestroyController(destroy_all_models) async def disable_user(self, username): @@ -255,7 +280,7 @@ class Controller(object): :param str username: Username """ - user_facade = client.UserManagerFacade.from_connection(self.connection) + user_facade = client.UserManagerFacade.from_connection(self.connection()) entity = client.Entity(tag.user(username)) return await user_facade.DisableUser([entity]) @@ -263,7 +288,7 @@ class Controller(object): """Re-enable a previously disabled user. """ - user_facade = client.UserManagerFacade.from_connection(self.connection) + user_facade = client.UserManagerFacade.from_connection(self.connection()) entity = client.Entity(tag.user(username)) return await user_facade.EnableUser([entity]) @@ -278,15 +303,33 @@ class Controller(object): """ Get the name of the cloud that this controller lives on. """ - cloud_facade = client.CloudFacade.from_connection(self.connection) + cloud_facade = client.CloudFacade.from_connection(self.connection()) result = await cloud_facade.Clouds() cloud = list(result.clouds.keys())[0] # only lives on one cloud return tag.untag('cloud-', cloud) - async def _model_uuids(self, all_=False, username=None): + async def get_models(self, all_=False, username=None): + """ + .. deprecated:: 0.7.0 + Use :meth:`.list_models` instead. + """ controller_facade = client.ControllerFacade.from_connection( self.connection) + for attempt in (1, 2, 3): + try: + return await controller_facade.AllModels() + except errors.JujuAPIError as e: + # retry concurrency error until resolved in Juju + # see: https://bugs.launchpad.net/juju/+bug/1721786 + if 'has been removed' not in e.message or attempt == 3: + raise + + async def model_uuids(self): + """Return a mapping of model names to UUIDs. + """ + controller_facade = client.ControllerFacade.from_connection( + self.connection()) for attempt in (1, 2, 3): try: response = await controller_facade.AllModels() @@ -297,17 +340,14 @@ class Controller(object): # see: https://bugs.launchpad.net/juju/+bug/1721786 if 'has been removed' not in e.message or attempt == 3: raise - await asyncio.sleep(attempt, loop=self.loop) + await asyncio.sleep(attempt, loop=self._connector.loop) - async def list_models(self, all_=False, username=None): + async def list_models(self): """Return list of names of the available models on this controller. - :param bool all_: List all models, regardless of user accessibilty - (admin use only) - :param str username: User for which to list models (admin use only) - + Equivalent to ``sorted((await self.model_uuids()).keys())`` """ - uuids = await self._model_uuids(all_, username) + uuids = await self.model_uuids() return sorted(uuids.keys()) def get_payloads(self, *patterns): @@ -347,24 +387,19 @@ class Controller(object): """Get a model by name or UUID. :param str model: Model name or UUID - + :returns Model: Connected Model instance. """ - uuids = await self._model_uuids() + uuids = await self.model_uuids() if model in uuids: - name_or_uuid = uuids[model] + uuid = uuids[model] else: - name_or_uuid = model + uuid = model + from juju.model import Model model = Model() - await model.connect( - self.connection.endpoint, - name_or_uuid, - self.connection.username, - self.connection.password, - self.connection.cacert, - self.connection.macaroons, - loop=self.loop, - ) + kwargs = self.connection().connect_params() + kwargs['uuid'] = uuid + await model._connect_direct(**kwargs) return model async def get_user(self, username): @@ -374,7 +409,7 @@ class Controller(object): :returns: A :class:`~juju.user.User` instance """ client_facade = client.UserManagerFacade.from_connection( - self.connection) + self.connection()) user = tag.user(username) args = [client.Entity(user)] try: @@ -396,32 +431,77 @@ class Controller(object): :returns: A list of :class:`~juju.user.User` instances """ client_facade = client.UserManagerFacade.from_connection( - self.connection) + self.connection()) response = await client_facade.UserInfo(None, include_disabled) return [User(self, r.result) for r in response.results] async def grant(self, username, acl='login'): - """Set access level of the given user on the controller - + """Grant access level of the given user on the controller. + Note that if the user already has higher permissions than the + provided ACL, this will do nothing (see revoke for a way to + remove permissions). :param str username: Username :param str acl: Access control ('login', 'add-model' or 'superuser') - + :returns: True if new access was granted, False if user already had + requested access or greater. Raises JujuError if failed. """ controller_facade = client.ControllerFacade.from_connection( - self.connection) + self.connection()) user = tag.user(username) - await self.revoke(username) changes = client.ModifyControllerAccess(acl, 'grant', user) - return await controller_facade.ModifyControllerAccess([changes]) + try: + await controller_facade.ModifyControllerAccess([changes]) + return True + except errors.JujuError as e: + if 'user already has' in str(e): + return False + else: + raise - async def revoke(self, username): - """Removes all access from a controller + async def revoke(self, username, acl='login'): + """Removes some or all access of a user to from a controller + If 'login' access is revoked, the user will no longer have any + permissions on the controller. Revoking a higher privilege from + a user without that privilege will have no effect. :param str username: username - + :param str acl: Access to remove ('login', 'add-model' or 'superuser') """ controller_facade = client.ControllerFacade.from_connection( - self.connection) + self.connection()) user = tag.user(username) changes = client.ModifyControllerAccess('login', 'revoke', user) return await controller_facade.ModifyControllerAccess([changes]) + + async def grant_model(self, username, model_uuid, acl='read'): + """Grant a user access to a model. Note that if the user + already has higher permissions than the provided ACL, + this will do nothing (see revoke_model for a way to remove permissions). + + :param str username: Username + :param str model_uuid: The UUID of the model to change. + :param str acl: Access control ('read, 'write' or 'admin') + """ + model_facade = client.ModelManagerFacade.from_connection( + self.connection()) + user = tag.user(username) + model = tag.model(model_uuid) + changes = client.ModifyModelAccess(acl, 'grant', model, user) + return await model_facade.ModifyModelAccess([changes]) + + async def revoke_model(self, username, model_uuid, acl='read'): + """Revoke some or all of a user's access to a model. + If 'read' access is revoked, the user will no longer have any + permissions on the model. Revoking a higher privilege from + a user without that privilege will have no effect. + + :param str username: Username to revoke + :param str model_uuid: The UUID of the model to change. + :param str acl: Access control ('read, 'write' or 'admin') + """ + model_facade = client.ModelManagerFacade.from_connection( + self.connection()) + user = tag.user(username) + model = tag.model(self.info.uuid) + changes = client.ModifyModelAccess(acl, 'revoke', model, user) + return await model_facade.ModifyModelAccess([changes]) diff --git a/modules/libjuju/juju/errors.py b/modules/libjuju/juju/errors.py index ecd1c0d..da11cdb 100644 --- a/modules/libjuju/juju/errors.py +++ b/modules/libjuju/juju/errors.py @@ -25,3 +25,25 @@ class JujuAPIError(JujuError): class JujuConnectionError(ConnectionError, JujuError): pass + + +class JujuAuthError(JujuConnectionError): + pass + + +class JujuRedirectException(Exception): + """Exception indicating that a redirection was requested""" + def __init__(self, redirect_info): + self.redirect_info = redirect_info + + @property + def ca_cert(self): + return self.redirect_info['ca-cert'] + + @property + def endpoints(self): + return [ + ('{value}:{port}'.format(**s), self.ca_cert) + for servers in self.redirect_info['servers'] + for s in servers if s['scope'] == 'public' + ] diff --git a/modules/libjuju/juju/machine.py b/modules/libjuju/juju/machine.py index 23b41c6..bd3d030 100644 --- a/modules/libjuju/juju/machine.py +++ b/modules/libjuju/juju/machine.py @@ -2,7 +2,7 @@ import asyncio import logging import os -from dateutil.parser import parse as parse_date +import pyrfc3339 from . import model, utils from .client import client @@ -66,8 +66,8 @@ class Machine(model.ModelEntity): change_log.append(('agent-version', '', agent_version)) # only update (other) delta fields if status data is newer - status_since = parse_date(machine['instance-status']['since']) - delta_since = parse_date(delta.data['instance-status']['since']) + status_since = pyrfc3339.parse(machine['instance-status']['since']) + delta_since = pyrfc3339.parse(delta.data['instance-status']['since']) if status_since > delta_since: for status_key in ('status', 'info', 'since'): delta_key = key_map[status_key] @@ -169,6 +169,8 @@ class Machine(model.ModelEntity): 'scp', '-i', os.path.expanduser('~/.local/share/juju/ssh/juju_id_rsa'), '-o', 'StrictHostKeyChecking=no', + '-q', + '-B', source, destination ] cmd += scp_opts.split() @@ -211,7 +213,7 @@ class Machine(model.ModelEntity): """Get the time when the `agent_status` was last updated. """ - return parse_date(self.safe_data['agent-status']['since']) + return pyrfc3339.parse(self.safe_data['agent-status']['since']) @property def agent_version(self): @@ -244,7 +246,7 @@ class Machine(model.ModelEntity): """Get the time when the `status` was last updated. """ - return parse_date(self.safe_data['instance-status']['since']) + return pyrfc3339.parse(self.safe_data['instance-status']['since']) @property def dns_name(self): @@ -260,3 +262,10 @@ class Machine(model.ModelEntity): if addresses: return addresses[0]['value'] return None + + @property + def series(self): + """Returns the series of the current machine + + """ + return self.safe_data['series'] diff --git a/modules/libjuju/juju/model.py b/modules/libjuju/juju/model.py index fc8d5e9..ac22599 100644 --- a/modules/libjuju/juju/model.py +++ b/modules/libjuju/juju/model.py @@ -14,26 +14,25 @@ from concurrent.futures import CancelledError from functools import partial from pathlib import Path -import websockets -import yaml import theblues.charmstore import theblues.errors +import websockets +import yaml from . import tag, utils -from .client import client -from .client import connection +from .client import client, connector from .client.client import ConfigValue -from .constraints import parse as parse_constraints, normalize_key -from .delta import get_entity_delta -from .delta import get_entity_class +from .constraints import parse as parse_constraints +from .constraints import normalize_key +from .delta import get_entity_class, get_entity_delta +from .errors import JujuAPIError, JujuError from .exceptions import DeadEntityException -from .errors import JujuError, JujuAPIError from .placement import parse as parse_placement log = logging.getLogger(__name__) -class _Observer(object): +class _Observer: """Wrapper around an observer callable. This wrapper allows filter criteria to be associated with the @@ -77,7 +76,7 @@ class _Observer(object): return True -class ModelObserver(object): +class ModelObserver: """ Base class for creating observers that react to changes in a model. """ @@ -100,7 +99,7 @@ class ModelObserver(object): pass -class ModelState(object): +class ModelState: """Holds the state of the model, including the delta history of all entities in the model. @@ -144,6 +143,14 @@ class ModelState(object): """ return self._live_entity_map('unit') + @property + def relations(self): + """Return a map of relation-id:Relation for all relations currently in + the model. + + """ + return self._live_entity_map('relation') + def entity_history(self, entity_type, entity_id): """Return the history deque for an entity. @@ -209,7 +216,7 @@ class ModelState(object): connected=connected) -class ModelEntity(object): +class ModelEntity: """An object in the Model tree""" def __init__(self, entity_id, model, history_index=-1, connected=True): @@ -228,7 +235,7 @@ class ModelEntity(object): self.model = model self._history_index = history_index self.connected = connected - self.connection = model.connection + self.connection = model.connection() def __repr__(self): return '<{} entity_id="{}">'.format(type(self).__name__, @@ -380,90 +387,148 @@ class ModelEntity(object): return self.model.state.get_entity(self.entity_type, self.entity_id) -class Model(object): +class Model: """ The main API for interacting with a Juju model. """ - def __init__(self, loop=None, - max_frame_size=connection.Connection.DEFAULT_FRAME_SIZE): - """Instantiate a new connected Model. + def __init__( + self, + loop=None, + max_frame_size=None, + bakery_client=None, + jujudata=None, + ): + """Instantiate a new Model. + + The connect method will need to be called before this + object can be used for anything interesting. + + If jujudata is None, jujudata.FileJujuData will be used. :param loop: an asyncio event loop :param max_frame_size: See `juju.client.connection.Connection.MAX_FRAME_SIZE` + :param bakery_client httpbakery.Client: The bakery client to use + for macaroon authorization. + :param jujudata JujuData: The source for current controller information. + """ + self._connector = connector.Connector( + loop=loop, + max_frame_size=max_frame_size, + bakery_client=bakery_client, + jujudata=jujudata, + ) + self._observers = weakref.WeakValueDictionary() + self.state = ModelState(self) + self._info = None + self._watch_stopping = asyncio.Event(loop=self._connector.loop) + self._watch_stopped = asyncio.Event(loop=self._connector.loop) + self._watch_received = asyncio.Event(loop=self._connector.loop) + self._watch_stopped.set() + self._charmstore = CharmStore(self._connector.loop) + + def is_connected(self): + """Reports whether the Model is currently connected.""" + return self._connector.is_connected() + + @property + def loop(self): + return self._connector.loop + def connection(self): + """Return the current Connection object. It raises an exception + if the Model is disconnected""" + return self._connector.connection() + + async def get_controller(self): + """Return a Controller instance for the currently connected model. + :return Controller: """ - self.loop = loop or asyncio.get_event_loop() - self.max_frame_size = max_frame_size - self.connection = None - self.observers = weakref.WeakValueDictionary() - self.state = ModelState(self) - self.info = None - self._watch_stopping = asyncio.Event(loop=self.loop) - self._watch_stopped = asyncio.Event(loop=self.loop) - self._watch_received = asyncio.Event(loop=self.loop) - self._charmstore = CharmStore(self.loop) + from juju.controller import Controller + controller = Controller(jujudata=self._connector.jujudata) + kwargs = self.connection().connect_params() + kwargs.pop('uuid') + await controller._connect_direct(**kwargs) + return controller async def __aenter__(self): - await self.connect_current() + await self.connect() return self async def __aexit__(self, exc_type, exc, tb): await self.disconnect() - if exc_type is not None: - return False + async def connect(self, model_name=None, **kwargs): + """Connect to a juju model. - async def connect(self, *args, **kw): - """Connect to an arbitrary Juju model. + If any arguments are specified other than model_name, then + model_name must be None and an explicit connection will be made + using Connection.connect using those parameters (the 'uuid' + parameter must be specified). - args and kw are passed through to Connection.connect() + Otherwise, if model_name is None, connect to the current model. - """ - if 'loop' not in kw: - kw['loop'] = self.loop - if 'max_frame_size' not in kw: - kw['max_frame_size'] = self.max_frame_size - self.connection = await connection.Connection.connect(*args, **kw) - await self._after_connect() + Otherwise, model_name must specify the name of a known + model. - async def connect_current(self): - """Connect to the current Juju model. + :param model_name: Format [controller:][user/]model """ - self.connection = await connection.Connection.connect_current( - self.loop, max_frame_size=self.max_frame_size) + await self.disconnect() + if not kwargs: + await self._connector.connect_model(model_name) + else: + if kwargs.get('uuid') is None: + raise ValueError('no UUID specified when connecting to model') + await self._connector.connect(**kwargs) await self._after_connect() async def connect_model(self, model_name): - """Connect to a specific Juju model by name. - - :param model_name: Format [controller:][user/]model + """ + .. deprecated:: 0.6.2 + Use connect(model_name=model_name) instead. + """ + return await self.connect(model_name=model_name) + async def connect_current(self): + """ + .. deprecated:: 0.6.2 + Use connect instead. """ - self.connection = await connection.Connection.connect_model( - model_name, self.loop, self.max_frame_size) + return await self.connect() + + async def _connect_direct(self, **kwargs): + await self.disconnect() + await self._connector.connect(**kwargs) await self._after_connect() async def _after_connect(self): - """Run initialization steps after connecting to websocket. - - """ self._watch() + + # Wait for the first packet of data from the AllWatcher, + # which contains all information on the model. + # TODO this means that we can't do anything until + # we've received all the model data, which might be + # a whole load of unneeded data if all the client wants + # to do is make one RPC call. await self._watch_received.wait() + await self.get_info() async def disconnect(self): """Shut down the watcher task and close websockets. """ - if self.connection and self.connection.is_open: + if not self._watch_stopped.is_set(): log.debug('Stopping watcher task') self._watch_stopping.set() await self._watch_stopped.wait() + self._watch_stopping.clear() + + if self.is_connected(): log.debug('Closing model connection') - await self.connection.close() - self.connection = None + await self._connector.disconnect() + self.info = None async def add_local_charm_dir(self, charm_dir, series): """Upload a local charm to the model. @@ -480,7 +545,7 @@ class Model(object): with fh: func = partial( self.add_local_charm, fh, series, os.stat(fh.name).st_size) - charm_url = await self.loop.run_in_executor(None, func) + charm_url = await self._connector.loop.run_in_executor(None, func) log.debug('Uploaded local charm: %s -> %s', charm_dir, charm_url) return charm_url @@ -505,7 +570,7 @@ class Model(object): instead. """ - conn, headers, path_prefix = self.connection.https_connection() + conn, headers, path_prefix = self.connection().https_connection() path = "%s/charms?series=%s" % (path_prefix, series) headers['Content-Type'] = 'application/zip' if size: @@ -549,13 +614,20 @@ class Model(object): async def block_until(self, *conditions, timeout=None, wait_period=0.5): """Return only after all conditions are true. + Raises `websockets.ConnectionClosed` if disconnected. """ - async def _block(): - while not all(c() for c in conditions): - if not (self.connection and self.connection.is_open): - raise websockets.ConnectionClosed(1006, 'no reason') - await asyncio.sleep(wait_period, loop=self.loop) - await asyncio.wait_for(_block(), timeout, loop=self.loop) + def _disconnected(): + return not (self.is_connected() and self.connection().is_open) + + def done(): + return _disconnected() or all(c() for c in conditions) + + await utils.block_until(done, + timeout=timeout, + wait_period=wait_period, + loop=self.loop) + if _disconnected(): + raise websockets.ConnectionClosed(1006, 'no reason') @property def applications(self): @@ -581,6 +653,13 @@ class Model(object): """ return self.state.units + @property + def relations(self): + """Return a list of all Relations currently in the model. + + """ + return list(self.state.relations.values()) + async def get_info(self): """Return a client.ModelInfo object for this Model. @@ -594,7 +673,7 @@ class Model(object): explicit call to this method. """ - facade = client.ClientFacade.from_connection(self.connection) + facade = client.ClientFacade.from_connection(self.connection()) self.info = await facade.ModelInfo() log.debug('Got ModelInfo: %s', vars(self.info)) @@ -639,7 +718,7 @@ class Model(object): """ observer = _Observer( callable_, entity_type, action, entity_id, predicate) - self.observers[observer] = callable_ + self._observers[observer] = callable_ def _watch(self): """Start an asynchronous watch against this model. @@ -650,13 +729,13 @@ class Model(object): async def _all_watcher(): try: allwatcher = client.AllWatcherFacade.from_connection( - self.connection) + self.connection()) while not self._watch_stopping.is_set(): try: results = await utils.run_with_interrupt( allwatcher.Next(), self._watch_stopping, - self.loop) + self._connector.loop) except JujuAPIError as e: if 'watcher was stopped' not in str(e): raise @@ -673,19 +752,27 @@ class Model(object): del allwatcher.Id continue except websockets.ConnectionClosed: - monitor = self.connection.monitor + monitor = self.connection().monitor if monitor.status == monitor.ERROR: # closed unexpectedly, try to reopen log.warning( 'Watcher: connection closed, reopening') - await self.connection.reconnect() + await self.connection().reconnect() + if monitor.status != monitor.CONNECTED: + # reconnect failed; abort and shutdown + log.error('Watcher: automatic reconnect ' + 'failed; stopping watcher') + break del allwatcher.Id continue else: # closed on request, go ahead and shutdown break if self._watch_stopping.is_set(): - await allwatcher.Stop() + try: + await allwatcher.Stop() + except websockets.ConnectionClosed: + pass # can't stop on a closed conn break for delta in results.deltas: delta = get_entity_delta(delta) @@ -704,7 +791,7 @@ class Model(object): self._watch_received.clear() self._watch_stopping.clear() self._watch_stopped.clear() - self.loop.create_task(_all_watcher()) + self._connector.loop.create_task(_all_watcher()) async def _notify_observers(self, delta, old_obj, new_obj): """Call observing callbacks, notifying them of a change in model state @@ -724,10 +811,10 @@ class Model(object): 'Model changed: %s %s %s', delta.entity, delta.type, delta.get_id()) - for o in self.observers: + for o in self._observers: if o.cares_about(delta): asyncio.ensure_future(o(delta, old_obj, new_obj, self), - loop=self.loop) + loop=self._connector.loop) async def _wait(self, entity_type, entity_id, action, predicate=None): """ @@ -744,7 +831,7 @@ class Model(object): has a 'completed' status. See the _Observer class for details. """ - q = asyncio.Queue(loop=self.loop) + q = asyncio.Queue(loop=self._connector.loop) async def callback(delta, old, new, model): await q.put(delta.get_id()) @@ -755,24 +842,19 @@ class Model(object): # 'remove' action return self.state._live_entity_map(entity_type).get(entity_id) - async def _wait_for_new(self, entity_type, entity_id=None, predicate=None): + async def _wait_for_new(self, entity_type, entity_id): """Wait for a new object to appear in the Model and return it. - Waits for an object of type ``entity_type`` with id ``entity_id``. - If ``entity_id`` is ``None``, it will wait for the first new entity - of the correct type. - - This coroutine blocks until the new object appears in the model. + Waits for an object of type ``entity_type`` with id ``entity_id`` + to appear in the model. This is similar to watching for the + object using ``block_until``, but uses the watcher rather than + polling. """ # if the entity is already in the model, just return it if entity_id in self.state._live_entity_map(entity_type): return self.state._live_entity_map(entity_type)[entity_id] - # if we know the entity_id, we can trigger on any action that puts - # the enitty into the model; otherwise, we have to watch for the - # next "add" action on that entity_type - action = 'add' if entity_id is None else None - return await self._wait(entity_type, entity_id, action, predicate) + return await self._wait(entity_type, entity_id, None) async def wait_for_action(self, action_id): """Given an action, wait for it to complete.""" @@ -785,7 +867,7 @@ class Model(object): def predicate(delta): return delta.data['status'] in ('completed', 'failed') - return await self._wait('action', action_id, 'change', predicate) + return await self._wait('action', action_id, None, predicate) async def add_machine( self, spec=None, constraints=None, disks=None, series=None): @@ -865,7 +947,7 @@ class Model(object): params.series = series # Submit the request. - client_facade = client.ClientFacade.from_connection(self.connection) + client_facade = client.ClientFacade.from_connection(self.connection()) results = await client_facade.AddMachines([params]) error = results.machines[0].error if error: @@ -881,29 +963,33 @@ class Model(object): :param str relation2: '[:]' """ - app_facade = client.ApplicationFacade.from_connection(self.connection) + app_facade = client.ApplicationFacade.from_connection(self.connection()) log.debug( 'Adding relation %s <-> %s', relation1, relation2) + def _find_relation(*specs): + for rel in self.relations: + if rel.matches(*specs): + return rel + return None + try: result = await app_facade.AddRelation([relation1, relation2]) except JujuAPIError as e: if 'relation already exists' not in e.message: raise - log.debug( - 'Relation %s <-> %s already exists', relation1, relation2) - # TODO: if relation already exists we should return the - # Relation ModelEntity here - return None + rel = _find_relation(relation1, relation2) + if rel: + return rel + raise JujuError('Relation {} {} exists but not in model'.format( + relation1, relation2)) - def predicate(delta): - endpoints = {} - for endpoint in delta.data['endpoints']: - endpoints[endpoint['application-name']] = endpoint['relation'] - return endpoints == result.endpoints + specs = ['{}:{}'.format(app, data['name']) + for app, data in result.endpoints.items()] - return await self._wait_for_new('relation', None, predicate) + await self.block_until(lambda: _find_relation(*specs) is not None) + return _find_relation(*specs) def add_space(self, name, *cidrs): """Add a new network space. @@ -924,7 +1010,7 @@ class Model(object): :param str key: The public ssh key """ - key_facade = client.KeyManagerFacade.from_connection(self.connection) + key_facade = client.KeyManagerFacade.from_connection(self.connection()) return await key_facade.AddKeys([key], user) add_ssh_keys = add_ssh_key @@ -1072,9 +1158,14 @@ class Model(object): for k, v in storage.items() } + entity_path = Path(entity_url.replace('local:', '')) + bundle_path = entity_path / 'bundle.yaml' + metadata_path = entity_path / 'metadata.yaml' + is_local = ( entity_url.startswith('local:') or - os.path.isdir(entity_url) + entity_path.is_dir() or + entity_path.is_file() ) if is_local: entity_id = entity_url.replace('local:', '') @@ -1082,10 +1173,11 @@ class Model(object): entity = await self.charmstore.entity(entity_url, channel=channel) entity_id = entity['Id'] - client_facade = client.ClientFacade.from_connection(self.connection) + client_facade = client.ClientFacade.from_connection(self.connection()) is_bundle = ((is_local and - (Path(entity_id) / 'bundle.yaml').exists()) or + (entity_id.endswith('.yaml') and entity_path.exists()) or + bundle_path.exists()) or (not is_local and 'bundle/' in entity_id)) if is_bundle: @@ -1100,9 +1192,9 @@ class Model(object): await asyncio.gather(*[ asyncio.ensure_future( self._wait_for_new('application', app_name), - loop=self.loop) + loop=self._connector.loop) for app_name in pending_apps - ], loop=self.loop) + ], loop=self._connector.loop) return [app for name, app in self.applications.items() if name in handler.applications] else: @@ -1118,6 +1210,9 @@ class Model(object): entity_id, entity) else: + if not application_name: + metadata = yaml.load(metadata_path.read_text()) + application_name = metadata['name'] # We have a local charm dir that needs to be uploaded charm_dir = os.path.abspath( os.path.expanduser(entity_id)) @@ -1163,7 +1258,7 @@ class Model(object): return None resources_facade = client.ResourcesFacade.from_connection( - self.connection) + self.connection()) response = await resources_facade.AddPendingResources( tag.application(application), entity_url, @@ -1186,7 +1281,7 @@ class Model(object): default_flow_style=False) app_facade = client.ApplicationFacade.from_connection( - self.connection) + self.connection()) app = client.ApplicationDeploy( charm_url=charm_url, @@ -1201,7 +1296,6 @@ class Model(object): storage=storage, placement=placement ) - result = await app_facade.Deploy([app]) errors = [r.error.message for r in result.results if r.error] if errors: @@ -1218,7 +1312,7 @@ class Model(object): """Destroy units by name. """ - app_facade = client.ApplicationFacade.from_connection(self.connection) + app_facade = client.ApplicationFacade.from_connection(self.connection()) log.debug( 'Destroying unit%s %s', @@ -1263,7 +1357,7 @@ class Model(object): which have `source` and `value` attributes. """ config_facade = client.ModelConfigFacade.from_connection( - self.connection + self.connection() ) result = await config_facade.ModelGet() config = result.config @@ -1277,22 +1371,6 @@ class Model(object): """ raise NotImplementedError() - async def grant(self, username, acl='read'): - """Grant a user access to this model. - - :param str username: Username - :param str acl: Access control ('read' or 'write') - - """ - controller_conn = await self.connection.controller() - model_facade = client.ModelManagerFacade.from_connection( - controller_conn) - user = tag.user(username) - model = tag.model(self.info.uuid) - changes = client.ModifyModelAccess(acl, 'grant', model, user) - await self.revoke(username) - return await model_facade.ModifyModelAccess([changes]) - def import_ssh_key(self, identity): """Add a public SSH key from a trusted indentity source to this model. @@ -1326,7 +1404,7 @@ class Model(object): else it's fingerprint """ - key_facade = client.KeyManagerFacade.from_connection(self.connection) + key_facade = client.KeyManagerFacade.from_connection(self.connection()) entity = {'tag': tag.model(self.info.uuid)} entities = client.Entities([entity]) return await key_facade.ListKeys(entities, raw_ssh) @@ -1399,7 +1477,7 @@ class Model(object): :param str user: Juju user to which the key is registered """ - key_facade = client.KeyManagerFacade.from_connection(self.connection) + key_facade = client.KeyManagerFacade.from_connection(self.connection()) key = base64.b64decode(bytes(key.strip().split()[1].encode('ascii'))) key = hashlib.md5(key).hexdigest() key = ':'.join(a + b for a, b in zip(key[::2], key[1::2])) @@ -1427,20 +1505,6 @@ class Model(object): """ raise NotImplementedError() - async def revoke(self, username): - """Revoke a user's access to this model. - - :param str username: Username to revoke - - """ - controller_conn = await self.connection.controller() - model_facade = client.ModelManagerFacade.from_connection( - controller_conn) - user = tag.user(username) - model = tag.model(self.info.uuid) - changes = client.ModifyModelAccess('read', 'revoke', model, user) - return await model_facade.ModifyModelAccess([changes]) - def run(self, command, timeout=None): """Run command on all machines in this model. @@ -1457,7 +1521,7 @@ class Model(object): `ConfigValue` instances, as returned by `get_config`. """ config_facade = client.ModelConfigFacade.from_connection( - self.connection + self.connection() ) for key, value in config.items(): if isinstance(value, ConfigValue): @@ -1506,7 +1570,7 @@ class Model(object): :param bool utc: Display time as UTC in RFC3339 format """ - client_facade = client.ClientFacade.from_connection(self.connection) + client_facade = client.ClientFacade.from_connection(self.connection()) return await client_facade.FullStatus(filters) def sync_tools( @@ -1587,7 +1651,7 @@ class Model(object): ', '.join(tags) if tags else "all units") metrics_facade = client.MetricsDebugFacade.from_connection( - self.connection) + self.connection()) entities = [client.Entity(tag) for tag in tags] metrics_result = await metrics_facade.GetMetrics(entities) @@ -1623,7 +1687,7 @@ def get_charm_series(path): return series[0] if series else None -class BundleHandler(object): +class BundleHandler: """ Handle bundles by using the API to translate bundle YAML into a plan of steps and then dispatching each of those using the API. @@ -1638,11 +1702,11 @@ class BundleHandler(object): app_units = self._units_by_app.setdefault(unit.application, []) app_units.append(unit_name) self.client_facade = client.ClientFacade.from_connection( - model.connection) + model.connection()) self.app_facade = client.ApplicationFacade.from_connection( - model.connection) + model.connection()) self.ann_facade = client.AnnotationsFacade.from_connection( - model.connection) + model.connection()) async def _handle_local_charms(self, bundle): """Search for references to local charms (i.e. filesystem paths) @@ -1694,8 +1758,11 @@ class BundleHandler(object): return bundle async def fetch_plan(self, entity_id): - is_local = not entity_id.startswith('cs:') and os.path.isdir(entity_id) - if is_local: + is_local = not entity_id.startswith('cs:') + + if is_local and os.path.isfile(entity_id): + bundle_yaml = Path(entity_id).read_text() + elif is_local and os.path.isdir(entity_id): bundle_yaml = (Path(entity_id) / "bundle.yaml").read_text() else: bundle_yaml = await self.charmstore.files(entity_id, @@ -1920,7 +1987,7 @@ class BundleHandler(object): return await entity.set_annotations(annotations) -class CharmStore(object): +class CharmStore: """ Async wrapper around theblues.charmstore.CharmStore """ @@ -1952,7 +2019,7 @@ class CharmStore(object): return wrapper -class CharmArchiveGenerator(object): +class CharmArchiveGenerator: """ Create a Zip archive of a local charm directory for upload to a controller. diff --git a/modules/libjuju/juju/relation.py b/modules/libjuju/juju/relation.py index ef8946a..d2f2053 100644 --- a/modules/libjuju/juju/relation.py +++ b/modules/libjuju/juju/relation.py @@ -5,7 +5,119 @@ from . import model log = logging.getLogger(__name__) +class Endpoint: + def __init__(self, model, data): + self.model = model + self.data = data + + def __repr__(self): + return ''.format(self.application.name, self.name) + + @property + def application(self): + return self.model.applications[self.data['application-name']] + + @property + def name(self): + return self.data['relation']['name'] + + @property + def interface(self): + return self.data['relation']['interface'] + + @property + def role(self): + return self.data['relation']['role'] + + @property + def scope(self): + return self.data['relation']['scope'] + + class Relation(model.ModelEntity): + def __repr__(self): + return ''.format(self.entity_id, self.key) + + @property + def endpoints(self): + return [Endpoint(self.model, data) + for data in self.safe_data['endpoints']] + + @property + def provides(self): + """ + The endpoint on the provides side of this relation, or None. + """ + for endpoint in self.endpoints: + if endpoint.role == 'provider': + return endpoint + return None + + @property + def requires(self): + """ + The endpoint on the requires side of this relation, or None. + """ + for endpoint in self.endpoints: + if endpoint.role == 'requirer': + return endpoint + return None + + @property + def peers(self): + """ + The peers endpoint of this relation, or None. + """ + for endpoint in self.endpoints: + if endpoint.role == 'peer': + return endpoint + return None + + @property + def is_subordinate(self): + return any(ep.scope == 'container' for ep in self.endpoints) + + @property + def is_peer(self): + return any(ep.role == 'peer' for ep in self.endpoints) + + def matches(self, *specs): + """ + Check if this relation matches relationship specs. + + Relation specs are strings that would be given to Juju to establish a + relation, and should be in the form ``[:]`` + where the ``:`` suffix is optional. If the suffix is + omitted, this relation will match on any endpoint as long as the given + application is involved. + + In other words, this relation will match a spec if that spec could have + created this relation. + + :return: True if all specs match. + """ + for spec in specs: + if ':' in spec: + app_name, endpoint_name = spec.split(':') + else: + app_name, endpoint_name = spec, None + for endpoint in self.endpoints: + if app_name == endpoint.application.name and \ + endpoint_name in (endpoint.name, None): + # found a match for this spec, so move to next one + break + else: + # no match for this spec + return False + return True + + @property + def applications(self): + """ + All applications involved in this relation. + """ + return [ep.application for ep in self.endpoints] + async def destroy(self): raise NotImplementedError() # TODO: destroy a relation diff --git a/modules/libjuju/juju/tag.py b/modules/libjuju/juju/tag.py index 2514229..319e8f8 100644 --- a/modules/libjuju/juju/tag.py +++ b/modules/libjuju/juju/tag.py @@ -1,3 +1,8 @@ +# TODO: Tags should be a proper class, so that we can distinguish whether +# something is already a tag or not. For example, 'user-foo' is a valid +# username, but is ambiguous with the already-tagged username 'foo'. + + def _prefix(prefix, s): if s and not s.startswith(prefix): return '{}{}'.format(prefix, s) diff --git a/modules/libjuju/juju/unit.py b/modules/libjuju/juju/unit.py index fc597bf..ce33b08 100644 --- a/modules/libjuju/juju/unit.py +++ b/modules/libjuju/juju/unit.py @@ -1,6 +1,6 @@ import logging -from dateutil.parser import parse as parse_date +import pyrfc3339 from . import model from .client import client @@ -21,7 +21,7 @@ class Unit(model.ModelEntity): """Get the time when the `agent_status` was last updated. """ - return parse_date(self.safe_data['agent-status']['since']) + return pyrfc3339.parse(self.safe_data['agent-status']['since']) @property def agent_status_message(self): @@ -42,7 +42,7 @@ class Unit(model.ModelEntity): """Get the time when the `workload_status` was last updated. """ - return parse_date(self.safe_data['workload-status']['since']) + return pyrfc3339.parse(self.safe_data['workload-status']['since']) @property def workload_status_message(self): diff --git a/juju/user.py b/modules/libjuju/juju/user.py similarity index 90% rename from juju/user.py rename to modules/libjuju/juju/user.py index b8890e1..01710d7 100644 --- a/juju/user.py +++ b/modules/libjuju/juju/user.py @@ -1,5 +1,6 @@ import logging -from dateutil.parser import parse as parse_date + +import pyrfc3339 from . import tag @@ -25,7 +26,7 @@ class User(object): @property def last_connection(self): - return parse_date(self._user_info.last_connection) + return pyrfc3339.parse(self._user_info.last_connection) @property def access(self): @@ -58,8 +59,8 @@ class User(object): :param str acl: Access control ('login', 'add-model', or 'superuser') """ - await self.controller.grant(self.username, acl) - self._user_info.access = acl + if await self.controller.grant(self.username, acl): + self._user_info.access = acl async def revoke(self): """Removes all access rights for this user from the controller. diff --git a/modules/libjuju/juju/utils.py b/modules/libjuju/juju/utils.py index 1d9bc1c..3565fd6 100644 --- a/modules/libjuju/juju/utils.py +++ b/modules/libjuju/juju/utils.py @@ -46,6 +46,7 @@ async def read_ssh_key(loop): can be passed on to a model. ''' + loop = loop or asyncio.get_event_loop() return await loop.run_in_executor(None, _read_ssh_key) @@ -71,6 +72,16 @@ class IdQueue: await queue.put(value) +async def block_until(*conditions, timeout=None, wait_period=0.5, loop=None): + """Return only after all conditions are true. + + """ + async def _block(): + while not all(c() for c in conditions): + await asyncio.sleep(wait_period, loop=loop) + await asyncio.wait_for(_block(), timeout, loop=loop) + + async def run_with_interrupt(task, event, loop=None): """ Awaits a task while allowing it to be interrupted by an `asyncio.Event`. @@ -91,6 +102,10 @@ async def run_with_interrupt(task, event, loop=None): return_when=asyncio.FIRST_COMPLETED) for f in pending: f.cancel() + exception = [f.exception() for f in done + if f is not event_task and f.exception()] + if exception: + raise exception[0] result = [f.result() for f in done if f is not event_task] if result: return result[0] diff --git a/modules/libjuju/tests/base.py b/modules/libjuju/tests/base.py index 96ed9c7..bae4b80 100644 --- a/modules/libjuju/tests/base.py +++ b/modules/libjuju/tests/base.py @@ -1,11 +1,12 @@ -import mock +import inspect import subprocess import uuid -import pytest - +import mock +from juju.client.jujudata import FileJujuData from juju.controller import Controller -from juju.client.connection import JujuData + +import pytest def is_bootstrapped(): @@ -14,60 +15,67 @@ def is_bootstrapped(): result.returncode == 0 and len(result.stdout.decode().strip()) > 0) + bootstrapped = pytest.mark.skipif( not is_bootstrapped(), reason='bootstrapped Juju environment required') +test_run_nonce = uuid.uuid4().hex[-4:] + class CleanController(): def __init__(self): - self.controller = None + self._controller = None async def __aenter__(self): - self.controller = Controller() - await self.controller.connect_current() - return self.controller + self._controller = Controller() + await self._controller.connect() + return self._controller async def __aexit__(self, exc_type, exc, tb): - await self.controller.disconnect() + await self._controller.disconnect() class CleanModel(): - def __init__(self): - self.user_name = None - self.controller = None - self.controller_name = None - self.model = None - self.model_name = None - self.model_uuid = None + def __init__(self, bakery_client=None): + self._controller = None + self._model = None + self._model_uuid = None + self._bakery_client = bakery_client async def __aenter__(self): - self.controller = Controller() - juju_data = JujuData() - self.controller_name = juju_data.current_controller() - self.user_name = juju_data.accounts()[self.controller_name]['user'] - await self.controller.connect_controller(self.controller_name) - - self.model_name = 'test-{}'.format(uuid.uuid4()) - self.model = await self.controller.add_model(self.model_name) + model_nonce = uuid.uuid4().hex[-4:] + frame = inspect.stack()[1] + test_name = frame.function.replace('_', '-') + jujudata = TestJujuData() + self._controller = Controller( + jujudata=jujudata, + bakery_client=self._bakery_client, + ) + controller_name = jujudata.current_controller() + user_name = jujudata.accounts()[controller_name]['user'] + await self._controller.connect(controller_name) + + model_name = 'test-{}-{}-{}'.format( + test_run_nonce, + test_name, + model_nonce, + ) + self._model = await self._controller.add_model(model_name) + + # Change the JujuData instance so that it will return the new + # model as the current model name, so that we'll connect + # to it by default. + jujudata.set_model( + controller_name, + user_name + "/" + model_name, + self._model.info.uuid, + ) # save the model UUID in case test closes model - self.model_uuid = self.model.info.uuid - - # Ensure that we connect to the new model by default. This also - # prevents failures if test was started with no current model. - self._patch_cm = mock.patch.object(JujuData, 'current_model', - return_value=self.model_name) - self._patch_cm.start() - - # Ensure that the models data includes this model, since it doesn't - # get added to the client store by Controller.add_model(). - self._orig_models = JujuData().models - self._patch_models = mock.patch.object(JujuData, 'models', - side_effect=self._models) - self._patch_models.start() + self._model_uuid = self._model.info.uuid - return self.model + return self._model def _models(self): result = self._orig_models() @@ -78,11 +86,36 @@ class CleanModel(): return result async def __aexit__(self, exc_type, exc, tb): - self._patch_models.stop() - self._patch_cm.stop() - await self.model.disconnect() - await self.controller.destroy_model(self.model_uuid) - await self.controller.disconnect() + await self._model.disconnect() + await self._controller.destroy_model(self._model_uuid) + await self._controller.disconnect() + + +class TestJujuData(FileJujuData): + def __init__(self): + self.__controller_name = None + self.__model_name = None + self.__model_uuid = None + super().__init__() + + def set_model(self, controller_name, model_name, model_uuid): + self.__controller_name = controller_name + self.__model_name = model_name + self.__model_uuid = model_uuid + + def current_model(self, *args, **kwargs): + return self.__model_name or super().current_model(*args, **kwargs) + + def models(self): + all_models = super().models() + if self.__model_name is None: + return all_models + all_models.setdefault(self.__controller_name, {}) + all_models[self.__controller_name].setdefault('models', {}) + cmodels = all_models[self.__controller_name]['models'] + cmodels[self.__model_name] = {'uuid': self.__model_uuid} + return all_models +>>>>>>> New N2VC interface + updated libjuju class AsyncMock(mock.MagicMock): diff --git a/modules/libjuju/tests/integration/test_client.py b/modules/libjuju/tests/integration/test_client.py index e4c9c92..240c471 100644 --- a/modules/libjuju/tests/integration/test_client.py +++ b/modules/libjuju/tests/integration/test_client.py @@ -1,7 +1,7 @@ -import pytest - from juju.client import client +import pytest + from .. import base @@ -9,7 +9,7 @@ from .. import base @pytest.mark.asyncio async def test_user_info(event_loop): async with base.CleanModel() as model: - controller_conn = await model.connection.controller() + controller_conn = await model.connection().controller() um = client.UserManagerFacade.from_connection(controller_conn) result = await um.UserInfo( diff --git a/modules/libjuju/tests/integration/test_connection.py b/modules/libjuju/tests/integration/test_connection.py index 290203d..79ad9d0 100644 --- a/modules/libjuju/tests/integration/test_connection.py +++ b/modules/libjuju/tests/integration/test_connection.py @@ -1,28 +1,19 @@ import asyncio -import pytest -from juju.client.connection import Connection from juju.client import client -from .. import base - +from juju.client.connection import Connection -@base.bootstrapped -@pytest.mark.asyncio -async def test_connect_current(event_loop): - async with base.CleanModel(): - conn = await Connection.connect_current() +import pytest - assert isinstance(conn, Connection) - await conn.close() +from .. import base @base.bootstrapped @pytest.mark.asyncio async def test_monitor(event_loop): - async with base.CleanModel(): - conn = await Connection.connect_current() - + async with base.CleanModel() as model: + conn = model.connection() assert conn.monitor.status == 'connected' await conn.close() @@ -33,15 +24,17 @@ async def test_monitor(event_loop): @pytest.mark.asyncio async def test_monitor_catches_error(event_loop): - async with base.CleanModel(): - conn = await Connection.connect_current() + async with base.CleanModel() as model: + conn = model.connection() assert conn.monitor.status == 'connected' - await conn.ws.close() - - assert conn.monitor.status == 'error' - - await conn.close() + try: + async with conn.monitor.reconnecting: + await conn.ws.close() + await asyncio.sleep(1) + assert conn.monitor.status == 'error' + finally: + await conn.close() @base.bootstrapped @@ -55,7 +48,7 @@ async def test_full_status(event_loop): channel='stable', ) - c = client.ClientFacade.from_connection(model.connection) + c = client.ClientFacade.from_connection(model.connection()) await c.FullStatus(None) @@ -64,15 +57,8 @@ async def test_full_status(event_loop): @pytest.mark.asyncio async def test_reconnect(event_loop): async with base.CleanModel() as model: - conn = await Connection.connect( - model.connection.endpoint, - model.connection.uuid, - model.connection.username, - model.connection.password, - model.connection.cacert, - model.connection.macaroons, - model.connection.loop, - model.connection.max_frame_size) + kwargs = model.connection().connect_params() + conn = await Connection.connect(**kwargs) try: await asyncio.sleep(0.1) assert conn.is_open diff --git a/modules/libjuju/tests/integration/test_controller.py b/modules/libjuju/tests/integration/test_controller.py index d559313..9c6f7ac 100644 --- a/modules/libjuju/tests/integration/test_controller.py +++ b/modules/libjuju/tests/integration/test_controller.py @@ -2,10 +2,13 @@ import asyncio import pytest import uuid -from .. import base -from juju.controller import Controller +from juju.client.connection import Connection from juju.errors import JujuAPIError +import pytest + +from .. import base + @base.bootstrapped @pytest.mark.asyncio @@ -57,14 +60,18 @@ async def test_change_user_password(event_loop): username = 'test-password{}'.format(uuid.uuid4()) user = await controller.add_user(username) await user.set_password('password') + # Check that we can connect with the new password. + new_connection = None try: - new_controller = Controller() - await new_controller.connect( - controller.connection.endpoint, username, 'password') + kwargs = controller.connection().connect_params() + kwargs['username'] = username + kwargs['password'] = 'password' + new_connection = await Connection.connect(**kwargs) except JujuAPIError: raise AssertionError('Unable to connect with new password') finally: - await new_controller.disconnect() + if new_connection: + await new_connection.close() @base.bootstrapped @@ -77,10 +84,10 @@ async def test_grant_revoke(event_loop): assert user.access == 'superuser' fresh = await controller.get_user(username) # fetch fresh copy assert fresh.access == 'superuser' - await user.grant('login') - assert user.access == 'login' + await user.grant('login') # already has 'superuser', so no-op + assert user.access == 'superuser' fresh = await controller.get_user(username) # fetch fresh copy - assert fresh.access == 'login' + assert fresh.access == 'superuser' await user.revoke() assert user.access is '' fresh = await controller.get_user(username) # fetch fresh copy @@ -120,6 +127,11 @@ async def test_get_model(event_loop): await controller.destroy_model(model_name) +async def _wait_for_model(controller, model_name): + while model_name not in await controller.list_models(): + await asyncio.sleep(0.5, loop=controller.loop) + + async def _wait_for_model_gone(controller, model_name): while model_name in await controller.list_models(): await asyncio.sleep(0.5, loop=controller.loop) @@ -132,6 +144,9 @@ async def test_destroy_model_by_name(event_loop): model_name = 'test-{}'.format(uuid.uuid4()) model = await controller.add_model(model_name) await model.disconnect() + await asyncio.wait_for(_wait_for_model(controller, + model_name), + timeout=60) await controller.destroy_model(model_name) await asyncio.wait_for(_wait_for_model_gone(controller, model_name), @@ -146,6 +161,9 @@ async def test_add_destroy_model_by_uuid(event_loop): model = await controller.add_model(model_name) model_uuid = model.info.uuid await model.disconnect() + await asyncio.wait_for(_wait_for_model(controller, + model_name), + timeout=60) await controller.destroy_model(model_uuid) await asyncio.wait_for(_wait_for_model_gone(controller, model_name), diff --git a/modules/libjuju/tests/integration/test_errors.py b/modules/libjuju/tests/integration/test_errors.py index 06b3826..b10dd06 100644 --- a/modules/libjuju/tests/integration/test_errors.py +++ b/modules/libjuju/tests/integration/test_errors.py @@ -40,7 +40,7 @@ async def test_juju_error_in_results_list(event_loop): from juju.client import client async with base.CleanModel() as model: - ann_facade = client.AnnotationsFacade.from_connection(model.connection) + ann_facade = client.AnnotationsFacade.from_connection(model.connection()) ann = client.EntityAnnotations( entity='badtag', @@ -58,11 +58,11 @@ async def test_juju_error_in_result(event_loop): looking at a single result coming back. ''' - from juju.errors import JujuError + from juju.errors import JujuError from juju.client import client async with base.CleanModel() as model: - app_facade = client.ApplicationFacade.from_connection(model.connection) + app_facade = client.ApplicationFacade.from_connection(model.connection()) with pytest.raises(JujuError): return await app_facade.GetCharmURL('foo') diff --git a/modules/libjuju/tests/integration/test_machine.py b/modules/libjuju/tests/integration/test_machine.py index cabf46d..8957ae1 100644 --- a/modules/libjuju/tests/integration/test_machine.py +++ b/modules/libjuju/tests/integration/test_machine.py @@ -1,8 +1,8 @@ import asyncio -import pytest - from tempfile import NamedTemporaryFile +import pytest + from .. import base @@ -42,6 +42,11 @@ async def test_status(event_loop): @base.bootstrapped @pytest.mark.asyncio async def test_scp(event_loop): + # ensure that asyncio.subprocess will work; + try: + asyncio.get_child_watcher().attach_loop(event_loop) + except RuntimeError: + pytest.skip('test_scp will always fail outside of MainThread') async with base.CleanModel() as model: await model.add_machine() await asyncio.wait_for( diff --git a/modules/libjuju/tests/integration/test_model.py b/modules/libjuju/tests/integration/test_model.py index 041f75a..ba2da92 100644 --- a/modules/libjuju/tests/integration/test_model.py +++ b/modules/libjuju/tests/integration/test_model.py @@ -1,11 +1,16 @@ import asyncio +import mock from concurrent.futures import ThreadPoolExecutor from pathlib import Path + +from juju.client.client import ConfigValue, ApplicationFacade +from juju.model import Model, ModelObserver +from juju.utils import block_until, run_with_interrupt + import pytest from .. import base -from juju.model import Model -from juju.client.client import ConfigValue + MB = 1 GB = 1024 @@ -18,14 +23,28 @@ async def test_deploy_local_bundle(event_loop): from pathlib import Path tests_dir = Path(__file__).absolute().parent.parent bundle_path = tests_dir / 'bundle' + mini_bundle_file_path = bundle_path / 'mini-bundle.yaml' async with base.CleanModel() as model: await model.deploy(str(bundle_path)) + await model.deploy(str(mini_bundle_file_path)) - for app in ('wordpress', 'mysql'): + for app in ('wordpress', 'mysql', 'myapp'): assert app in model.applications +@base.bootstrapped +@pytest.mark.asyncio +async def test_deploy_local_charm(event_loop): + from pathlib import Path + tests_dir = Path(__file__).absolute().parent.parent + charm_path = tests_dir / 'charm' + + async with base.CleanModel() as model: + await model.deploy(str(charm_path)) + assert 'charm' in model.applications + + @base.bootstrapped @pytest.mark.asyncio async def test_deploy_bundle(event_loop): @@ -43,7 +62,7 @@ async def test_deploy_channels_revs(event_loop): charm = 'cs:~johnsca/libjuju-test' stable = await model.deploy(charm, 'a1') edge = await model.deploy(charm, 'a2', channel='edge') - rev = await model.deploy(charm+'-2', 'a3') + rev = await model.deploy(charm + '-2', 'a3') assert [a.charm_url for a in (stable, edge, rev)] == [ 'cs:~johnsca/libjuju-test-1', @@ -111,17 +130,44 @@ async def test_relate(event_loop): # subordinates must be deployed without units num_units=0, ) - my_relation = await model.add_relation( - 'ubuntu', - 'nrpe', - ) + + relation_added = asyncio.Event() + timeout = asyncio.Event() + + class TestObserver(ModelObserver): + async def on_relation_add(self, delta, old, new, model): + if set(new.key.split()) == {'nrpe:general-info', + 'ubuntu:juju-info'}: + relation_added.set() + event_loop.call_later(2, timeout.set) + + model.add_observer(TestObserver()) + + real_app_facade = ApplicationFacade.from_connection(model.connection()) + mock_app_facade = mock.MagicMock() + + async def mock_AddRelation(*args): + # force response delay from AddRelation to test race condition + # (see https://github.com/juju/python-libjuju/issues/191) + result = await real_app_facade.AddRelation(*args) + await relation_added.wait() + return result + + mock_app_facade.AddRelation = mock_AddRelation + + with mock.patch.object(ApplicationFacade, 'from_connection', + return_value=mock_app_facade): + my_relation = await run_with_interrupt(model.add_relation( + 'ubuntu', + 'nrpe', + ), timeout, event_loop) assert isinstance(my_relation, Relation) -async def _deploy_in_loop(new_loop, model_name): - new_model = Model(new_loop) - await new_model.connect_model(model_name) +async def _deploy_in_loop(new_loop, model_name, jujudata): + new_model = Model(new_loop, jujudata=jujudata) + await new_model.connect(model_name) try: await new_model.deploy('cs:xenial/ubuntu') assert 'ubuntu' in new_model.applications @@ -138,7 +184,7 @@ async def test_explicit_loop_threaded(event_loop): with ThreadPoolExecutor(1) as executor: f = executor.submit( new_loop.run_until_complete, - _deploy_in_loop(new_loop, model_name)) + _deploy_in_loop(new_loop, model_name, model._connector.jujudata)) f.result() await model._wait_for_new('application', 'ubuntu') assert 'ubuntu' in model.applications @@ -155,7 +201,7 @@ async def test_store_resources_charm(event_loop): lambda: ( len(ghost.units) > 0 and ghost.units[0].workload_status in terminal_statuses) - ) + ) # ghost will go in to blocked (or error, for older # charm revs) if the resource is missing assert ghost.units[0].workload_status == 'active' @@ -174,7 +220,7 @@ async def test_store_resources_bundle(event_loop): lambda: ( len(ghost.units) > 0 and ghost.units[0].workload_status in terminal_statuses) - ) + ) # ghost will go in to blocked (or error, for older # charm revs) if the resource is missing assert ghost.units[0].workload_status == 'active' @@ -206,9 +252,8 @@ async def test_get_machines(event_loop): @pytest.mark.asyncio async def test_watcher_reconnect(event_loop): async with base.CleanModel() as model: - await model.connection.ws.close() - await asyncio.sleep(0.1) - assert model.connection.is_open + await model.connection().ws.close() + await block_until(model.is_connected, timeout=3) @base.bootstrapped diff --git a/modules/libjuju/tests/integration/test_unit.py b/modules/libjuju/tests/integration/test_unit.py index 1604c31..8b2251c 100644 --- a/modules/libjuju/tests/integration/test_unit.py +++ b/modules/libjuju/tests/integration/test_unit.py @@ -1,8 +1,8 @@ import asyncio -import pytest - from tempfile import NamedTemporaryFile +import pytest + from .. import base @@ -52,6 +52,11 @@ async def test_run_action(event_loop): @base.bootstrapped @pytest.mark.asyncio async def test_scp(event_loop): + # ensure that asyncio.subprocess will work; + try: + asyncio.get_child_watcher().attach_loop(event_loop) + except RuntimeError: + pytest.skip('test_scp will always fail outside of MainThread') async with base.CleanModel() as model: app = await model.deploy('ubuntu') diff --git a/modules/libjuju/tests/unit/test_client.py b/modules/libjuju/tests/unit/test_client.py index e9fde8e..42134df 100644 --- a/modules/libjuju/tests/unit/test_client.py +++ b/modules/libjuju/tests/unit/test_client.py @@ -5,7 +5,6 @@ Tests for generated client code import mock - from juju.client import client diff --git a/modules/libjuju/tests/unit/test_connection.py b/modules/libjuju/tests/unit/test_connection.py index f69b8d6..0925d84 100644 --- a/modules/libjuju/tests/unit/test_connection.py +++ b/modules/libjuju/tests/unit/test_connection.py @@ -1,13 +1,14 @@ import asyncio import json -import mock -import pytest from collections import deque +import mock +from juju.client.connection import Connection from websockets.exceptions import ConnectionClosed +import pytest + from .. import base -from juju.client.connection import Connection class WebsocketMock: @@ -31,7 +32,6 @@ class WebsocketMock: @pytest.mark.asyncio async def test_out_of_order(event_loop): - con = Connection(*[None]*4) ws = WebsocketMock([ {'request-id': 1}, {'request-id': 3}, @@ -42,13 +42,24 @@ async def test_out_of_order(event_loop): {'request-id': 2}, {'request-id': 3}, ] - con._get_sll = mock.MagicMock() + minimal_facades = [{'name': 'Pinger', 'versions': [1]}] + con = None try: - with mock.patch('websockets.connect', base.AsyncMock(return_value=ws)): - await con.open() + with \ + mock.patch('websockets.connect', base.AsyncMock(return_value=ws)), \ + mock.patch( + 'juju.client.connection.Connection.login', + base.AsyncMock(return_value={'response': { + 'facades': minimal_facades, + }}), + ), \ + mock.patch('juju.client.connection.Connection._get_ssl'), \ + mock.patch('juju.client.connection.Connection._pinger', base.AsyncMock()): + con = await Connection.connect('0.1.2.3:999') actual_responses = [] for i in range(3): actual_responses.append(await con.rpc({'version': 1})) assert actual_responses == expected_responses finally: - await con.close() + if con: + await con.close() diff --git a/modules/libjuju/tests/unit/test_constraints.py b/modules/libjuju/tests/unit/test_constraints.py index cb9d773..00b9156 100644 --- a/modules/libjuju/tests/unit/test_constraints.py +++ b/modules/libjuju/tests/unit/test_constraints.py @@ -6,6 +6,7 @@ import unittest from juju import constraints + class TestConstraints(unittest.TestCase): def test_mem_regex(self): diff --git a/modules/libjuju/tests/unit/test_loop.py b/modules/libjuju/tests/unit/test_loop.py index f12368e..9043df6 100644 --- a/modules/libjuju/tests/unit/test_loop.py +++ b/modules/libjuju/tests/unit/test_loop.py @@ -1,5 +1,6 @@ import asyncio import unittest + import juju.loop @@ -15,6 +16,7 @@ class TestLoop(unittest.TestCase): def test_run(self): assert asyncio.get_event_loop() == self.loop + async def _test(): return 'success' self.assertEqual(juju.loop.run(_test()), 'success') diff --git a/modules/libjuju/tests/unit/test_model.py b/modules/libjuju/tests/unit/test_model.py index 222d881..2e33236 100644 --- a/modules/libjuju/tests/unit/test_model.py +++ b/modules/libjuju/tests/unit/test_model.py @@ -1,8 +1,11 @@ import unittest import mock + import asynctest +from juju.client.jujudata import FileJujuData + def _make_delta(entity, type_, data=None): from juju.client.client import Delta @@ -68,8 +71,8 @@ class TestModelState(unittest.TestCase): from juju.model import Model from juju.application import Application - loop = mock.MagicMock() - model = Model(loop=loop) + model = Model() + model._connector = mock.MagicMock() delta = _make_delta('application', 'add', dict(name='foo')) # test add @@ -118,7 +121,7 @@ def test_get_series(): class TestContextManager(asynctest.TestCase): @asynctest.patch('juju.model.Model.disconnect') - @asynctest.patch('juju.model.Model.connect_current') + @asynctest.patch('juju.model.Model.connect') async def test_normal_use(self, mock_connect, mock_disconnect): from juju.model import Model @@ -129,7 +132,7 @@ class TestContextManager(asynctest.TestCase): self.assertTrue(mock_disconnect.called) @asynctest.patch('juju.model.Model.disconnect') - @asynctest.patch('juju.model.Model.connect_current') + @asynctest.patch('juju.model.Model.connect') async def test_exception(self, mock_connect, mock_disconnect): from juju.model import Model @@ -143,13 +146,57 @@ class TestContextManager(asynctest.TestCase): self.assertTrue(mock_connect.called) self.assertTrue(mock_disconnect.called) - @asynctest.patch('juju.client.connection.JujuData.current_controller') - async def test_no_current_connection(self, mock_current_controller): + async def test_no_current_connection(self): from juju.model import Model from juju.errors import JujuConnectionError - mock_current_controller.return_value = "" + class NoControllerJujuData(FileJujuData): + def current_controller(self): + return "" with self.assertRaises(JujuConnectionError): - async with Model(): + async with Model(jujudata=NoControllerJujuData()): pass + + +class TestModelConnect(asynctest.TestCase): + @asynctest.patch('juju.client.connector.Connector.connect_model') + @asynctest.patch('juju.model.Model._after_connect') + async def test_model_connect_no_args(self, mock_after_connect, mock_connect_model): + from juju.model import Model + m = Model() + await m.connect() + mock_connect_model.assert_called_once_with(None) + + @asynctest.patch('juju.client.connector.Connector.connect_model') + @asynctest.patch('juju.model.Model._after_connect') + async def test_model_connect_with_model_name(self, mock_after_connect, mock_connect_model): + from juju.model import Model + m = Model() + await m.connect(model_name='foo') + mock_connect_model.assert_called_once_with('foo') + + @asynctest.patch('juju.client.connector.Connector.connect_model') + @asynctest.patch('juju.model.Model._after_connect') + async def test_model_connect_with_endpoint_but_no_uuid( + self, + mock_after_connect, + mock_connect_model, + ): + from juju.model import Model + m = Model() + with self.assertRaises(ValueError): + await m.connect(endpoint='0.1.2.3:4566') + self.assertEqual(mock_connect_model.call_count, 0) + + @asynctest.patch('juju.client.connector.Connector.connect') + @asynctest.patch('juju.model.Model._after_connect') + async def test_model_connect_with_endpoint_and_uuid( + self, + mock_after_connect, + mock_connect, + ): + from juju.model import Model + m = Model() + await m.connect(endpoint='0.1.2.3:4566', uuid='some-uuid') + mock_connect.assert_called_once_with(endpoint='0.1.2.3:4566', uuid='some-uuid') diff --git a/modules/libjuju/tests/unit/test_overrides.py b/modules/libjuju/tests/unit/test_overrides.py index 6485408..a5835ff 100644 --- a/modules/libjuju/tests/unit/test_overrides.py +++ b/modules/libjuju/tests/unit/test_overrides.py @@ -1,6 +1,6 @@ -import pytest +from juju.client.overrides import Binary, Number # noqa -from juju.client.overrides import Number, Binary # noqa +import pytest # test cases ported from: diff --git a/modules/libjuju/tests/unit/test_placement.py b/modules/libjuju/tests/unit/test_placement.py index a78a28d..5a933ec 100644 --- a/modules/libjuju/tests/unit/test_placement.py +++ b/modules/libjuju/tests/unit/test_placement.py @@ -5,7 +5,7 @@ import unittest from juju import placement -from juju.client import client + class TestPlacement(unittest.TestCase): diff --git a/n2vc/__init__.py b/n2vc/__init__.py new file mode 100644 index 0000000..93353d0 --- /dev/null +++ b/n2vc/__init__.py @@ -0,0 +1 @@ +version = '0.0.1' diff --git a/n2vc/vnf.py b/n2vc/vnf.py new file mode 100644 index 0000000..c606dda --- /dev/null +++ b/n2vc/vnf.py @@ -0,0 +1,686 @@ + +import logging +import os +import os.path +import re +import ssl +import sys +import time + +# FIXME: this should load the juju inside or modules without having to +# explicitly install it. Check why it's not working. +# Load our subtree of the juju library +path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +path = os.path.join(path, "modules/libjuju/") +if path not in sys.path: + sys.path.insert(1, path) + +from juju.controller import Controller +from juju.model import Model, ModelObserver + + +# We might need this to connect to the websocket securely, but test and verify. +try: + ssl._create_default_https_context = ssl._create_unverified_context +except AttributeError: + # Legacy Python doesn't verify by default (see pep-0476) + # https://www.python.org/dev/peps/pep-0476/ + pass + + +# Custom exceptions +class JujuCharmNotFound(Exception): + """The Charm can't be found or is not readable.""" + + +class JujuApplicationExists(Exception): + """The Application already exists.""" + + +# Quiet the debug logging +logging.getLogger('websockets.protocol').setLevel(logging.INFO) +logging.getLogger('juju.client.connection').setLevel(logging.WARN) +logging.getLogger('juju.model').setLevel(logging.WARN) +logging.getLogger('juju.machine').setLevel(logging.WARN) + +class VCAMonitor(ModelObserver): + """Monitor state changes within the Juju Model.""" + callback = None + callback_args = None + log = None + ns_name = None + application_name = None + + def __init__(self, ns_name, application_name, callback, *args): + self.log = logging.getLogger(__name__) + + self.ns_name = ns_name + self.application_name = application_name + self.callback = callback + self.callback_args = args + + async def on_change(self, delta, old, new, model): + """React to changes in the Juju model.""" + + if delta.entity == "unit": + try: + if old and new: + old_status = old.workload_status + new_status = new.workload_status + if old_status == new_status: + """The workload status may fluctuate around certain events, + so wait until the status has stabilized before triggering + the callback.""" + if self.callback: + self.callback( + self.ns_name, + self.application_name, + new_status, + *self.callback_args) + except Exception as e: + self.log.debug("[1] notify_callback exception {}".format(e)) + + +######## +# TODO +# +# Create unique models per network service +# Document all public functions + +class N2VC: + + # Juju API + api = None + log = None + controller = None + connecting = False + authenticated = False + + models = {} + default_model = None + + # Model Observers + monitors = {} + + # VCA config + hostname = "" + port = 17070 + username = "" + secret = "" + + def __init__(self, + log=None, + server='127.0.0.1', + port=17070, + user='admin', + secret=None, + artifacts=None + ): + """Initialize N2VC + + :param vcaconfig dict A dictionary containing the VCA configuration + + :param artifacts str The directory where charms required by a vnfd are + stored. + + :Example: + n2vc = N2VC(vcaconfig={ + 'secret': 'MzI3MDJhOTYxYmM0YzRjNTJiYmY1Yzdm', + 'user': 'admin', + 'ip-address': '10.44.127.137', + 'port': 17070, + 'artifacts': '/path/to/charms' + }) + + """ + + if log: + self.log = log + else: + self.log = logging.getLogger(__name__) + + # Quiet websocket traffic + logging.getLogger('websockets.protocol').setLevel(logging.INFO) + logging.getLogger('juju.client.connection').setLevel(logging.WARN) + logging.getLogger('model').setLevel(logging.WARN) + # logging.getLogger('websockets.protocol').setLevel(logging.DEBUG) + + self.log.debug('JujuApi: instantiated') + + self.server = server + self.port = port + + self.secret = secret + if user.startswith('user-'): + self.user = user + else: + self.user = 'user-{}'.format(user) + + self.endpoint = '%s:%d' % (server, int(port)) + + self.artifacts = artifacts + + def __del__(self): + """Close any open connections.""" + yield self.logout() + + def notify_callback(self, model_name, application_name, status, callback=None, *callback_args): + try: + if callback: + callback(model_name, application_name, status, *callback_args) + except Exception as e: + self.log.error("[0] notify_callback exception {}".format(e)) + return True + + # Public methods + async def CreateNetworkService(self, nsd): + """Create a new model to encapsulate this network service. + + Create a new model in the Juju controller to encapsulate the + charms associated with a network service. + + You can pass either the nsd record or the id of the network + service, but this method will fail without one of them. + """ + if not self.authenticated: + await self.login() + + # Ideally, we will create a unique model per network service. + # This change will require all components, i.e., LCM and SO, to use + # N2VC for 100% compatibility. If we adopt unique models for the LCM, + # services deployed via LCM would't be manageable via SO and vice versa + + return self.default_model + + async def DeployCharms(self, model_name, application_name, vnfd, charm_path, params={}, machine_spec={}, callback=None, *callback_args): + """Deploy one or more charms associated with a VNF. + + Deploy the charm(s) referenced in a VNF Descriptor. + + You can pass either the nsd record or the id of the network + service, but this method will fail without one of them. + + :param str ns_name: The name of the network service + :param str application_name: The name of the application + :param dict vnfd: The name of the application + :param str charm_path: The path to the Juju charm + :param dict params: A dictionary of runtime parameters + Examples:: + { + 'rw_mgmt_ip': '1.2.3.4' + } + :param dict machine_spec: A dictionary describing the machine to install to + Examples:: + { + 'hostname': '1.2.3.4', + 'username': 'ubuntu', + } + :param obj callback: A callback function to receive status changes. + :param tuple callback_args: A list of arguments to be passed to the callback + """ + + ######################################################## + # Verify the path to the charm exists and is readable. # + ######################################################## + if not os.path.exists(charm_path): + self.log.debug("Charm path doesn't exist: {}".format(charm_path)) + self.notify_callback(model_name, application_name, "failed", callback, *callback_args) + raise JujuCharmNotFound("No artifacts configured.") + + ################################ + # Login to the Juju controller # + ################################ + if not self.authenticated: + self.log.debug("Authenticating with Juju") + await self.login() + + ########################################## + # Get the model for this network service # + ########################################## + # TODO: In a point release, we will use a model per deployed network + # service. In the meantime, we will always use the 'default' model. + model_name = 'default' + model = await self.get_model(model_name) + # if model_name not in self.models: + # self.log.debug("Getting model {}".format(model_name)) + # self.models[model_name] = await self.controller.get_model(model_name) + # model = await self.CreateNetworkService(ns_name) + + ################################################### + # Get the name of the charm and its configuration # + ################################################### + config_dict = vnfd['vnf-configuration'] + juju = config_dict['juju'] + charm = juju['charm'] + self.log.debug("Charm: {}".format(charm)) + + ######################################## + # Verify the application doesn't exist # + ######################################## + app = await self.get_application(model, application_name) + if app: + raise JujuApplicationExists("Can't deploy application \"{}\" to model \"{}\" because it already exists.".format(application_name, model)) + + ############################################################ + # Create a monitor to watch for application status changes # + ############################################################ + if callback: + self.log.debug("Setting monitor<->callback") + self.monitors[application_name] = VCAMonitor(model_name, application_name, callback, *callback_args) + model.add_observer(self.monitors[application_name]) + + + ######################################################## + # Check for specific machine placement (native charms) # + ######################################################## + to = "" + if machine_spec.keys(): + # TODO: This needs to be tested. + # if all(k in machine_spec for k in ['hostname', 'username']): + # # Enlist the existing machine in Juju + # machine = await self.model.add_machine(spec='ssh:%@%'.format( + # specs['host'], + # specs['user'], + # )) + # to = machine.id + pass + + ####################################### + # Get the initial charm configuration # + ####################################### + + rw_mgmt_ip = None + if 'rw_mgmt_ip' in params: + rw_mgmt_ip = params['rw_mgmt_ip'] + + initial_config = self._get_config_from_dict( + config_dict['initial-config-primitive'], + {'': rw_mgmt_ip} + ) + + self.log.debug("JujuApi: Deploying charm {} ({}) from {}".format( + charm, + application_name, + charm_path, + to=to, + )) + + ######################################################## + # Deploy the charm and apply the initial configuration # + ######################################################## + app = await model.deploy( + charm_path, + application_name=application_name, + series='xenial', + config=initial_config, + to=None, + ) + + async def ExecutePrimitive(self, model_name, application_name, primitive, callback, *callback_args, **params): + try: + if not self.authenticated: + await self.login() + + # FIXME: This is hard-coded until model-per-ns is added + model_name = 'default' + + if primitive == 'config': + # config is special, and expecting params to be a dictionary + await self.set_config(application_name, params['params']) + else: + model = await self.controller.get_model(model_name) + app = await self.get_application(model, application_name) + if app: + # Run against the first (and probably only) unit in the app + unit = app.units[0] + if unit: + self.log.debug("Executing primitive {}".format(primitive)) + action = await unit.run_action(primitive, **params) + action = await action.wait() + await model.disconnect() + except Exception as e: + self.log.debug("Caught exception while executing primitive: {}".format(e)) + raise e + + async def RemoveCharms(self, model_name, application_name, callback=None, *callback_args): + try: + if not self.authenticated: + await self.login() + + model = await self.get_model(model_name) + app = await self.get_application(model, application_name) + if app: + self.notify_callback(model_name, application_name, "removing", callback, *callback_args) + await app.remove() + self.notify_callback(model_name, application_name, "removed", callback, *callback_args) + except Exception as e: + print("Caught exception: {}".format(e)) + raise e + + async def DestroyNetworkService(self, nsd): + raise NotImplementedError() + + async def GetMetrics(self, nsd, vnfd): + """Get the metrics collected by the VCA.""" + raise NotImplementedError() + + # Non-public methods + async def add_relation(self, a, b, via=None): + """ + Add a relation between two application endpoints. + + :param a An application endpoint + :param b An application endpoint + :param via The egress subnet(s) for outbound traffic, e.g., + (192.168.0.0/16,10.0.0.0/8) + """ + if not self.authenticated: + await self.login() + + m = await self.get_model() + try: + m.add_relation(a, b, via) + finally: + await m.disconnect() + + async def apply_config(self, config, application): + """Apply a configuration to the application.""" + print("JujuApi: Applying configuration to {}.".format( + application + )) + return await self.set_config(application=application, config=config) + + def _get_config_from_dict(self, config_primitive, values): + """Transform the yang config primitive to dict.""" + config = {} + for primitive in config_primitive: + if primitive['name'] == 'config': + for parameter in primitive['parameter']: + param = str(parameter['name']) + if parameter['value'] == "": + config[param] = str(values[parameter['value']]) + else: + config[param] = str(parameter['value']) + + return config + + def _get_config_from_yang(self, config_primitive, values): + """Transform the yang config primitive to dict.""" + config = {} + for primitive in config_primitive.values(): + if primitive['name'] == 'config': + for parameter in primitive['parameter'].values(): + param = str(parameter['name']) + if parameter['value'] == "": + config[param] = str(values[parameter['value']]) + else: + config[param] = str(parameter['value']) + + return config + + def FormatApplicationName(self, *args): + """ + Generate a Juju-compatible Application name + + :param args tuple: Positional arguments to be used to construct the + application name. + + Limitations:: + - Only accepts characters a-z and non-consequitive dashes (-) + - Application name should not exceed 50 characters + + Examples:: + + FormatApplicationName("ping_pong_ns", "ping_vnf", "a") + """ + + appname = "" + for c in "-".join(list(args)): + if c.isdigit(): + c = chr(97 + int(c)) + elif not c.isalpha(): + c = "-" + appname += c + return re.sub('\-+', '-', appname.lower()) + + + # def format_application_name(self, nsd_name, vnfr_name, member_vnf_index=0): + # """Format the name of the application + # + # Limitations: + # - Only accepts characters a-z and non-consequitive dashes (-) + # - Application name should not exceed 50 characters + # """ + # name = "{}-{}-{}".format(nsd_name, vnfr_name, member_vnf_index) + # new_name = '' + # for c in name: + # if c.isdigit(): + # c = chr(97 + int(c)) + # elif not c.isalpha(): + # c = "-" + # new_name += c + # return re.sub('\-+', '-', new_name.lower()) + + def format_model_name(self, name): + """Format the name of model. + + Model names may only contain lowercase letters, digits and hyphens + """ + + return name.replace('_', '-').lower() + + async def get_application(self, model, application): + """Get the deployed application.""" + if not self.authenticated: + await self.login() + + app = None + if application and model: + if model.applications: + if application in model.applications: + app = model.applications[application] + + return app + + async def get_model(self, model_name='default'): + """Get a model from the Juju Controller. + + Note: Model objects returned must call disconnected() before it goes + out of scope.""" + if not self.authenticated: + await self.login() + + if model_name not in self.models: + print("connecting to model {}".format(model_name)) + self.models[model_name] = await self.controller.get_model(model_name) + + return self.models[model_name] + + async def login(self): + """Login to the Juju controller.""" + + if self.authenticated: + return + + self.connecting = True + + self.log.debug("JujuApi: Logging into controller") + + cacert = None + self.controller = Controller() + + if self.secret: + self.log.debug("Connecting to controller... ws://{}:{} as {}/{}".format(self.endpoint, self.port, self.user, self.secret)) + await self.controller.connect( + endpoint=self.endpoint, + username=self.user, + password=self.secret, + cacert=cacert, + ) + else: + # current_controller no longer exists + # self.log.debug("Connecting to current controller...") + # await self.controller.connect_current() + self.log.fatal("VCA credentials not configured.") + + self.authenticated = True + self.log.debug("JujuApi: Logged into controller") + + # self.default_model = await self.controller.get_model("default") + + async def logout(self): + """Logout of the Juju controller.""" + if not self.authenticated: + return + + try: + if self.default_model: + self.log.debug("Disconnecting model {}".format(self.default_model)) + await self.default_model.disconnect() + self.default_model = None + + for model in self.models: + await self.models[model].disconnect() + + if self.controller: + self.log.debug("Disconnecting controller {}".format(self.controller)) + await self.controller.disconnect() + # self.controller = None + + self.authenticated = False + except Exception as e: + self.log.fail("Fatal error logging out of Juju Controller: {}".format(e)) + raise e + + + # async def remove_application(self, name): + # """Remove the application.""" + # if not self.authenticated: + # await self.login() + # + # app = await self.get_application(name) + # if app: + # self.log.debug("JujuApi: Destroying application {}".format( + # name, + # )) + # + # await app.destroy() + + async def remove_relation(self, a, b): + """ + Remove a relation between two application endpoints + + :param a An application endpoint + :param b An application endpoint + """ + if not self.authenticated: + await self.login() + + m = await self.get_model() + try: + m.remove_relation(a, b) + finally: + await m.disconnect() + + async def resolve_error(self, application=None): + """Resolve units in error state.""" + if not self.authenticated: + await self.login() + + app = await self.get_application(self.default_model, application) + if app: + self.log.debug("JujuApi: Resolving errors for application {}".format( + application, + )) + + for unit in app.units: + app.resolved(retry=True) + + async def run_action(self, application, action_name, **params): + """Execute an action and return an Action object.""" + if not self.authenticated: + await self.login() + result = { + 'status': '', + 'action': { + 'tag': None, + 'results': None, + } + } + app = await self.get_application(self.default_model, application) + if app: + # We currently only have one unit per application + # so use the first unit available. + unit = app.units[0] + + self.log.debug("JujuApi: Running Action {} against Application {}".format( + action_name, + application, + )) + + action = await unit.run_action(action_name, **params) + + # Wait for the action to complete + await action.wait() + + result['status'] = action.status + result['action']['tag'] = action.data['id'] + result['action']['results'] = action.results + + return result + + async def set_config(self, application, config): + """Apply a configuration to the application.""" + if not self.authenticated: + await self.login() + + app = await self.get_application(self.default_model, application) + if app: + self.log.debug("JujuApi: Setting config for Application {}".format( + application, + )) + await app.set_config(config) + + # Verify the config is set + newconf = await app.get_config() + for key in config: + if config[key] != newconf[key]['value']: + self.log.debug("JujuApi: Config not set! Key {} Value {} doesn't match {}".format(key, config[key], newconf[key])) + + async def set_parameter(self, parameter, value, application=None): + """Set a config parameter for a service.""" + if not self.authenticated: + await self.login() + + self.log.debug("JujuApi: Setting {}={} for Application {}".format( + parameter, + value, + application, + )) + return await self.apply_config( + {parameter: value}, + application=application, + ) + + async def wait_for_application(self, name, timeout=300): + """Wait for an application to become active.""" + if not self.authenticated: + await self.login() + + app = await self.get_application(self.default_model, name) + if app: + self.log.debug( + "JujuApi: Waiting {} seconds for Application {}".format( + timeout, + name, + ) + ) + + await self.default_model.block_until( + lambda: all( + unit.agent_status == 'idle' + and unit.workload_status + in ['active', 'unknown'] for unit in app.units + ), + timeout=timeout + ) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9c558e3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +. diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..a4cceb1 --- /dev/null +++ b/setup.py @@ -0,0 +1,40 @@ +# Copyright 2016 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path +from setuptools import setup, find_packages + +setup( + name='N2VC', + version=0.1, + packages=find_packages( + exclude=["*.tests", "*.tests.*", "tests.*", "tests"]), + install_requires=[ + 'macaroonbakery>=1.1,<2.0', + 'pyRFC3339>=1.0,<2.0', + 'pyyaml>=3.0,<4.0', + 'theblues>=0.3.8,<1.0', + 'websockets>=4.0,<5.0', + ], + include_package_data=True, + maintainer='', + maintainer_email='', + description=(''), + url='', + license='Apache 2', + entry_points={ + 'console_scripts': [ + ], + }, +) diff --git a/tests/test_async_task.py b/tests/test_async_task.py new file mode 100644 index 0000000..da6e96e --- /dev/null +++ b/tests/test_async_task.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +# Recreate the conditions in which this will be used in OSM, called via tasks +import asyncio + +if __name__ == "__main__": + main() + +async def do_something(): + pass + +def main(): + + loop = asyncio.get_event_loop() + loop.run_until_complete(do_something()) + loop.close() + loop = None diff --git a/tests/test_python.py b/tests/test_python.py new file mode 100755 index 0000000..b770a01 --- /dev/null +++ b/tests/test_python.py @@ -0,0 +1,434 @@ +# A simple test to exercise the libraries' functionality +import asyncio +import functools +import os +import sys +import logging +import osm_im.vnfd as vnfd_catalog +import osm_im.nsd as nsd_catalog +from pyangbind.lib.serialise import pybindJSONDecoder +import unittest +import yaml +from n2vc.vnf import N2VC + +NSD_YAML = """ +nsd-catalog: + nsd: + - id: rift_ping_pong_ns + logo: rift_logo.png + name: ping_pong_ns + short-name: ping_pong_ns + vendor: RIFT.io + version: '1.1' + description: RIFT.io sample ping pong network service + constituent-vnfd: + - member-vnf-index: '1' + vnfd-id-ref: rift_ping_vnf + - member-vnf-index: '2' + vnfd-id-ref: rift_pong_vnf + initial-service-primitive: + - name: start traffic + parameter: + - name: port + value: 5555 + - name: ssh-username + value: fedora + - name: ssh-password + value: fedora + seq: '1' + user-defined-script: start_traffic.py + input-parameter-xpath: + - xpath: /nsd:nsd-catalog/nsd:nsd/nsd:vendor + ip-profiles: + - description: Inter VNF Link + ip-profile-params: + gateway-address: 31.31.31.210 + ip-version: ipv4 + subnet-address: 31.31.31.0/24 + dhcp-params: + count: 200 + start-address: 31.31.31.2 + name: InterVNFLink + placement-groups: + - member-vnfd: + - member-vnf-index-ref: '1' + vnfd-id-ref: rift_ping_vnf + - member-vnf-index-ref: '2' + vnfd-id-ref: rift_pong_vnf + name: Orcus + requirement: Place this VM on the Kuiper belt object Orcus + strategy: COLOCATION + - member-vnfd: + - member-vnf-index-ref: '1' + vnfd-id-ref: rift_ping_vnf + - member-vnf-index-ref: '2' + vnfd-id-ref: rift_pong_vnf + name: Quaoar + requirement: Place this VM on the Kuiper belt object Quaoar + strategy: COLOCATION + vld: + - id: mgmt_vl + description: Management VL + name: mgmt_vl + short-name: mgmt_vl + vim-network-name: mgmt + type: ELAN + vendor: RIFT.io + version: '1.0' + mgmt-network: 'true' + vnfd-connection-point-ref: + - member-vnf-index-ref: '1' + vnfd-connection-point-ref: ping_vnfd/cp0 + vnfd-id-ref: rift_ping_vnf + - member-vnf-index-ref: '2' + vnfd-connection-point-ref: pong_vnfd/cp0 + vnfd-id-ref: rift_pong_vnf + - id: ping_pong_vl1 + description: Data VL + ip-profile-ref: InterVNFLink + name: data_vl + short-name: data_vl + type: ELAN + vendor: RIFT.io + version: '1.0' + vnfd-connection-point-ref: + - member-vnf-index-ref: '1' + vnfd-connection-point-ref: ping_vnfd/cp1 + vnfd-id-ref: rift_ping_vnf + - member-vnf-index-ref: '2' + vnfd-connection-point-ref: pong_vnfd/cp1 + vnfd-id-ref: rift_pong_vnf +""" + +VNFD_VCA_YAML = """ +vnfd-catalog: + vnfd: + - id: rift_ping_vnf + name: ping_vnf + short-name: ping_vnf + logo: rift_logo.png + vendor: RIFT.io + version: '1.1' + description: This is an example RIFT.ware VNF + connection-point: + - name: ping_vnfd/cp0 + type: VPORT + - name: ping_vnfd/cp1 + type: VPORT + http-endpoint: + - path: api/v1/ping/stats + port: '18888' + mgmt-interface: + dashboard-params: + path: api/v1/ping/stats + port: '18888' + port: '18888' + cp: ping_vnfd/cp0 + placement-groups: + - member-vdus: + - member-vdu-ref: iovdu_0 + name: Eris + requirement: Place this VM on the Kuiper belt object Eris + strategy: COLOCATION + vdu: + - cloud-init-file: ping_cloud_init.cfg + count: '1' + interface: + - name: eth0 + position: 0 + type: EXTERNAL + virtual-interface: + type: VIRTIO + external-connection-point-ref: ping_vnfd/cp0 + - name: eth1 + position: 1 + type: EXTERNAL + virtual-interface: + type: VIRTIO + external-connection-point-ref: ping_vnfd/cp1 + id: iovdu_0 + image: Fedora-x86_64-20-20131211.1-sda-ping.qcow2 + name: iovdu_0 + vm-flavor: + memory-mb: '512' + storage-gb: '4' + vcpu-count: '1' + vnf-configuration: + config-primitive: + - name: start + - name: stop + - name: restart + - name: config + parameter: + - data-type: STRING + default-value: + name: ssh-hostname + - data-type: STRING + default-value: fedora + name: ssh-username + - data-type: STRING + default-value: fedora + name: ssh-password + - data-type: STRING + name: ssh-private-key + - data-type: STRING + default-value: ping + name: mode + read-only: 'true' + - name: set-server + parameter: + - data-type: STRING + name: server-ip + - data-type: INTEGER + name: server-port + - name: set-rate + parameter: + - data-type: INTEGER + default-value: '5' + name: rate + - name: start-traffic + - name: stop-traffic + initial-config-primitive: + - name: config + parameter: + - name: ssh-hostname + value: + - name: ssh-username + value: fedora + - name: ssh-password + value: fedora + - name: mode + value: ping + seq: '1' + - name: start + seq: '2' + juju: + charm: pingpong +""" + +NSD_DICT = {'name': 'ping_pong_ns', 'short-name': 'ping_pong_ns', 'ip-profiles': [{'ip-profile-params': {'ip-version': 'ipv4', 'dhcp-params': {'count': 200, 'start-address': '31.31.31.2'}, 'subnet-address': '31.31.31.0/24', 'gateway-address': '31.31.31.210'}, 'description': 'Inter VNF Link', 'name': 'InterVNFLink'}], 'logo': 'rift_logo.png', 'description': 'RIFT.io sample ping pong network service', '_admin': {'storage': {'folder': 'd9bc2e64-dfec-4c36-a8bb-c4667e8d7a53', 'tarfile': 'pkg', 'file': 'ping_pong_ns', 'path': '/app/storage/', 'fs': 'local'}, 'created': 1521127984.6414561, 'modified': 1521127984.6414561, 'projects_write': ['admin'], 'projects_read': ['admin']}, 'placement-groups': [{'member-vnfd': [{'member-vnf-index-ref': '1', 'vnfd-id-ref': 'rift_ping_vnf'}, {'member-vnf-index-ref': '2', 'vnfd-id-ref': 'rift_pong_vnf'}], 'name': 'Orcus', 'strategy': 'COLOCATION', 'requirement': 'Place this VM on the Kuiper belt object Orcus'}, {'member-vnfd': [{'member-vnf-index-ref': '1', 'vnfd-id-ref': 'rift_ping_vnf'}, {'member-vnf-index-ref': '2', 'vnfd-id-ref': 'rift_pong_vnf'}], 'name': 'Quaoar', 'strategy': 'COLOCATION', 'requirement': 'Place this VM on the Kuiper belt object Quaoar'}], 'input-parameter-xpath': [{'xpath': '/nsd:nsd-catalog/nsd:nsd/nsd:vendor'}], 'version': '1.1', 'vld': [{'name': 'mgmt', 'short-name': 'mgmt', 'mgmt-network': 'true', 'vim-network-name': 'mgmt', 'version': '1.0', 'vnfd-connection-point-ref': [{'vnfd-connection-point-ref': 'ping_vnfd/cp0', 'member-vnf-index-ref': '1', 'vnfd-id-ref': 'rift_ping_vnf'}, {'vnfd-connection-point-ref': 'pong_vnfd/cp0', 'member-vnf-index-ref': '2', 'vnfd-id-ref': 'rift_pong_vnf'}], 'description': 'Management VL', 'vendor': 'RIFT.io', 'type': 'ELAN', 'id': 'mgmt'}, {'ip-profile-ref': 'InterVNFLink', 'name': 'data_vl', 'short-name': 'data_vl', 'version': '1.0', 'vnfd-connection-point-ref': [{'vnfd-connection-point-ref': 'ping_vnfd/cp1', 'member-vnf-index-ref': '1', 'vnfd-id-ref': 'rift_ping_vnf'}, {'vnfd-connection-point-ref': 'pong_vnfd/cp1', 'member-vnf-index-ref': '2', 'vnfd-id-ref': 'rift_pong_vnf'}], 'description': 'Data VL', 'vendor': 'RIFT.io', 'type': 'ELAN', 'id': 'ping_pong_vl1'}], 'constituent-vnfd': [{'member-vnf-index': '1', 'vnfd-id-ref': 'rift_ping_vnf'}, {'member-vnf-index': '2', 'vnfd-id-ref': 'rift_pong_vnf'}], '_id': 'd9bc2e64-dfec-4c36-a8bb-c4667e8d7a53', 'vendor': 'RIFT.io', 'id': 'rift_ping_pong_ns', 'initial-service-primitive': [{'parameter': [{'name': 'port', 'value': 5555}, {'name': 'ssh-username', 'value': 'fedora'}, {'name': 'ssh-password', 'value': 'fedora'}], 'name': 'start traffic', 'seq': '1', 'user-defined-script': 'start_traffic.py'}]} + + +VNFD_PING_DICT = {'name': 'ping_vnf', 'short-name': 'ping_vnf', 'mgmt-interface': {'port': '18888', 'cp': 'ping_vnfd/cp0', 'dashboard-params': {'port': '18888', 'path': 'api/v1/ping/stats'}}, 'description': 'This is an example RIFT.ware VNF', 'connection-point': [{'type': 'VPORT', 'name': 'ping_vnfd/cp0'}, {'type': 'VPORT', 'name': 'ping_vnfd/cp1'}], '_admin': {'storage': {'folder': '9ad8de93-cfcc-4da9-9795-d7dc5fec184e', 'fs': 'local', 'file': 'ping_vnf', 'path': '/app/storage/', 'tarfile': 'pkg'}, 'created': 1521127972.1878572, 'modified': 1521127972.1878572, 'projects_write': ['admin'], 'projects_read': ['admin']}, 'vnf-configuration': {'initial-config-primitive': [{'parameter': [{'name': 'ssh-hostname', 'value': ''}, {'name': 'ssh-username', 'value': 'ubuntu'}, {'name': 'ssh-password', 'value': 'ubuntu'}, {'name': 'mode', 'value': 'ping'}], 'name': 'config', 'seq': '1'}, {'name': 'start', 'seq': '2'}], 'juju': {'charm': 'pingpong'}, 'config-primitive': [{'name': 'start'}, {'name': 'stop'}, {'name': 'restart'}, {'parameter': [{'data-type': 'STRING', 'name': 'ssh-hostname', 'default-value': ''}, {'data-type': 'STRING', 'name': 'ssh-username', 'default-value': 'ubuntu'}, {'data-type': 'STRING', 'name': 'ssh-password', 'default-value': 'ubuntu'}, {'data-type': 'STRING', 'name': 'ssh-private-key'}, {'data-type': 'STRING', 'name': 'mode', 'read-only': 'true', 'default-value': 'ping'}], 'name': 'config'}, {'parameter': [{'data-type': 'STRING', 'name': 'server-ip'}, {'data-type': 'INTEGER', 'name': 'server-port'}], 'name': 'set-server'}, {'parameter': [{'data-type': 'INTEGER', 'name': 'rate', 'default-value': '5'}], 'name': 'set-rate'}, {'name': 'start-traffic'}, {'name': 'stop-traffic'}]}, 'placement-groups': [{'member-vdus': [{'member-vdu-ref': 'iovdu_0'}], 'name': 'Eris', 'strategy': 'COLOCATION', 'requirement': 'Place this VM on the Kuiper belt object Eris'}], 'http-endpoint': [{'port': '18888', 'path': 'api/v1/ping/stats'}], 'version': '1.1', 'logo': 'rift_logo.png', 'vdu': [{'vm-flavor': {'vcpu-count': '1', 'storage-gb': '20', 'memory-mb': '2048'}, 'count': '1', 'interface': [{'type': 'EXTERNAL', 'name': 'eth0', 'position': 0, 'virtual-interface': {'type': 'VIRTIO'}, 'external-connection-point-ref': 'ping_vnfd/cp0'}, {'type': 'EXTERNAL', 'name': 'eth1', 'position': 1, 'virtual-interface': {'type': 'VIRTIO'}, 'external-connection-point-ref': 'ping_vnfd/cp1'}], 'name': 'iovdu_0', 'image': 'xenial', 'id': 'iovdu_0', 'cloud-init-file': 'ping_cloud_init.cfg'}], '_id': '9ad8de93-cfcc-4da9-9795-d7dc5fec184e', 'vendor': 'RIFT.io', 'id': 'rift_ping_vnf'} + +VNFD_PONG_DICT = {'name': 'pong_vnf', 'short-name': 'pong_vnf', 'mgmt-interface': {'port': '18888', 'cp': 'pong_vnfd/cp0', 'dashboard-params': {'port': '18888', 'path': 'api/v1/pong/stats'}}, 'description': 'This is an example RIFT.ware VNF', 'connection-point': [{'type': 'VPORT', 'name': 'pong_vnfd/cp0'}, {'type': 'VPORT', 'name': 'pong_vnfd/cp1'}], '_admin': {'storage': {'folder': '9ad8de93-cfcc-4da9-9795-d7dc5fec184e', 'fs': 'local', 'file': 'pong_vnf', 'path': '/app/storage/', 'tarfile': 'pkg'}, 'created': 1521127972.1878572, 'modified': 1521127972.1878572, 'projects_write': ['admin'], 'projects_read': ['admin']}, 'vnf-configuration': {'initial-config-primitive': [{'parameter': [{'name': 'ssh-hostname', 'value': ''}, {'name': 'ssh-username', 'value': 'ubuntu'}, {'name': 'ssh-password', 'value': 'ubuntu'}, {'name': 'mode', 'value': 'pong'}], 'name': 'config', 'seq': '1'}, {'name': 'start', 'seq': '2'}], 'juju': {'charm': 'pingpong'}, 'config-primitive': [{'name': 'start'}, {'name': 'stop'}, {'name': 'restart'}, {'parameter': [{'data-type': 'STRING', 'name': 'ssh-hostname', 'default-value': ''}, {'data-type': 'STRING', 'name': 'ssh-username', 'default-value': 'ubuntu'}, {'data-type': 'STRING', 'name': 'ssh-password', 'default-value': 'ubuntu'}, {'data-type': 'STRING', 'name': 'ssh-private-key'}, {'data-type': 'STRING', 'name': 'mode', 'read-only': 'true', 'default-value': 'pong'}], 'name': 'config'}, {'parameter': [{'data-type': 'STRING', 'name': 'server-ip'}, {'data-type': 'INTEGER', 'name': 'server-port'}], 'name': 'set-server'}, {'parameter': [{'data-type': 'INTEGER', 'name': 'rate', 'default-value': '5'}], 'name': 'set-rate'}, {'name': 'start-traffic'}, {'name': 'stop-traffic'}]}, 'placement-groups': [{'member-vdus': [{'member-vdu-ref': 'iovdu_0'}], 'name': 'Eris', 'strategy': 'COLOCATION', 'requirement': 'Place this VM on the Kuiper belt object Eris'}], 'http-endpoint': [{'port': '18888', 'path': 'api/v1/pong/stats'}], 'version': '1.1', 'logo': 'rift_logo.png', 'vdu': [{'vm-flavor': {'vcpu-count': '1', 'storage-gb': '20', 'memory-mb': '2048'}, 'count': '1', 'interface': [{'type': 'EXTERNAL', 'name': 'eth0', 'position': 0, 'virtual-interface': {'type': 'VIRTIO'}, 'external-connection-point-ref': 'ping_vnfd/cp0'}, {'type': 'EXTERNAL', 'name': 'eth1', 'position': 1, 'virtual-interface': {'type': 'VIRTIO'}, 'external-connection-point-ref': 'ping_vnfd/cp1'}], 'name': 'iovdu_0', 'image': 'xenial', 'id': 'iovdu_0', 'cloud-init-file': 'pong_cloud_init.cfg'}], '_id': '9ad8de93-cfcc-4da9-9795-d7dc5fec184e', 'vendor': 'RIFT.io', 'id': 'rift_pong_vnf'} + + +class PythonTest(unittest.TestCase): + n2vc = None + + def setUp(self): + + self.log = logging.getLogger() + self.log.level = logging.DEBUG + + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(None) + + # Extract parameters from the environment in order to run our test + vca_host = os.getenv('VCA_HOST', '127.0.0.1') + vca_port = os.getenv('VCA_PORT', 17070) + vca_user = os.getenv('VCA_USER', 'admin') + vca_charms = os.getenv('VCA_CHARMS', None) + vca_secret = os.getenv('VCA_SECRET', None) + self.n2vc = N2VC( + log=self.log, + server=vca_host, + port=vca_port, + user=vca_user, + secret=vca_secret, + artifacts=vca_charms, + ) + + def tearDown(self): + self.loop.run_until_complete(self.n2vc.logout()) + + def get_vnf_descriptor(self, descriptor): + vnfd = vnfd_catalog.vnfd() + try: + data = yaml.load(descriptor) + pybindJSONDecoder.load_ietf_json(data, None, None, obj=vnfd) + except ValueError: + assert False + return vnfd + + def get_ns_descriptor(self, descriptor): + nsd = nsd_catalog.nsd() + try: + data = yaml.load(descriptor) + pybindJSONDecoder.load_ietf_json(data, None, None, obj=nsd) + except ValueError: + assert False + return nsd + + # def test_descriptor(self): + # """Test loading and parsing a descriptor.""" + # nsd = self.get_ns_descriptor(NSD_YAML) + # vnfd = self.get_vnf_descriptor(VNFD_VCA_YAML) + # if vnfd and nsd: + # pass + + # def test_yang_to_dict(self): + # # Test the conversion of the native object returned by pybind to a dict + # # Extract parameters from the environment in order to run our test + # + # nsd = self.get_ns_descriptor(NSD_YAML) + # new = yang_to_dict(nsd) + # self.assertEqual(NSD_DICT, new) + + def n2vc_callback(self, model_name, application_name, workload_status, task=None): + """We pass the vnfd when setting up the callback, so expect it to be + returned as a tuple.""" + if workload_status and not task: + self.log.debug("Callback: workload status \"{}\"".format(workload_status)) + + if workload_status in ["blocked"]: + task = asyncio.ensure_future( + self.n2vc.ExecutePrimitive( + model_name, + application_name, + "config", + None, + params={ + 'ssh-hostname': '10.195.8.78', + 'ssh-username': 'ubuntu', + 'ssh-password': 'ubuntu' + } + ) + ) + task.add_done_callback(functools.partial(self.n2vc_callback, None, None, None)) + pass + elif workload_status in ["active"]: + self.log.debug("Removing charm") + task = asyncio.ensure_future( + self.n2vc.RemoveCharms(model_name, application_name, self.n2vc_callback, model_name, application_name) + ) + task.add_done_callback(functools.partial(self.n2vc_callback, None, None, None)) + elif task: + if task.done(): + self.loop.stop() + + def test_deploy_application(self): + stream_handler = logging.StreamHandler(sys.stdout) + self.log.addHandler(stream_handler) + try: + self.log.info("Log handler installed") + nsd = NSD_DICT + vnfd_ping = VNFD_PING_DICT + vnfd_pong = VNFD_PONG_DICT + if nsd and vnfd_ping and vnfd_pong: + vca_charms = os.getenv('VCA_CHARMS', None) + + config = vnfd_ping['vnf-configuration'] + self.assertIsNotNone(config) + + juju = config['juju'] + self.assertIsNotNone(juju) + + charm = juju['charm'] + self.assertIsNotNone(charm) + + charm_dir = "{}/{}".format(vca_charms, charm) + # n.callback_status = self.callback_status + + # Setting this to an IP that will fail the initial config. + # This will be detected in the callback, which will execute + # the "config" primitive with the right IP address. + params = { + 'rw_mgmt_ip': '127.0.0.1' + } + + # self.loop.run_until_complete(n.CreateNetworkService(nsd)) + # task = asyncio.ensure_future(n.DeployCharms(vnfd, nsd=nsd, artifacts=charm_dir)) + # task = asyncio.ensure_future(n.DeployCharms(vnfd, nsd=nsd, artifacts=charm_dir)) + ns_name = "default" + + ping_vnf_name = self.n2vc.FormatApplicationName(ns_name, vnfd_ping['name']) + pong_vnf_name = self.n2vc.FormatApplicationName(ns_name, vnfd_pong['name']) + + self.loop.run_until_complete(self.n2vc.DeployCharms(ns_name, ping_vnf_name, vnfd_ping, charm_dir, params, {}, self.n2vc_callback)) + self.loop.run_until_complete(self.n2vc.DeployCharms(ns_name, pong_vnf_name, vnfd_pong, charm_dir, params, {}, self.n2vc_callback)) + self.loop.run_forever() + + # self.loop.run_until_complete(n.GetMetrics(vnfd, nsd=nsd)) + # Test actions + # ExecutePrimitive(self, nsd, vnfd, vnf_member_index, primitive, callback, *callback_args, **params): + + # self.loop.run_until_complete(n.DestroyNetworkService(nsd)) + + # self.loop.run_until_complete(self.n2vc.logout()) + finally: + self.log.removeHandler(stream_handler) + + # def test_deploy_application(self): + # + # nsd = self.get_ns_descriptor(NSD_YAML) + # vnfd = self.get_vnf_descriptor(VNFD_VCA_YAML) + # if nsd and vnfd: + # # 1) Test that we're parsing the data correctly + # for vnfd_yang in vnfd.vnfd_catalog.vnfd.itervalues(): + # vnfd_rec = vnfd_yang.get() + # # Each vnfd may have a charm + # # print(vnfd) + # config = vnfd_rec.get('vnf-configuration') + # self.assertIsNotNone(config) + # + # juju = config.get('juju') + # self.assertIsNotNone(juju) + # + # charm = juju.get('charm') + # self.assertIsNotNone(charm) + # + # # 2) Exercise the data by deploying the charms + # + # # Extract parameters from the environment in order to run our test + # vca_host = os.getenv('VCA_HOST', '127.0.0.1') + # vca_port = os.getenv('VCA_PORT', 17070) + # vca_user = os.getenv('VCA_USER', 'admin') + # vca_charms = os.getenv('VCA_CHARMS', None) + # vca_secret = os.getenv('VCA_SECRET', None) + # # n = N2VC( + # # server='10.195.8.254', + # # port=17070, + # # user='admin', + # # secret='74e7aa0cc9cb294de3af294bd76b4604' + # # ) + # n = N2VC( + # server=vca_host, + # port=vca_port, + # user=vca_user, + # secret=vca_secret, + # artifacts=vca_charms, + # ) + # + # n.callback_status = self.callback_status + # + # self.loop.run_until_complete(n.CreateNetworkService(nsd)) + # self.loop.run_until_complete(n.DeployCharms(vnfd, nsd=nsd)) + # # self.loop.run_until_complete(n.GetMetrics(vnfd, nsd=nsd)) + # + # # self.loop.run_until_complete(n.RemoveCharms(nsd, vnfd)) + # self.loop.run_until_complete(n.DestroyNetworkService(nsd)) + # + # self.loop.run_until_complete(n.logout()) + # n = None + + def callback_status(self, nsd, vnfd, workload_status): + """An example callback. + + This is an example of how a client using N2VC can receive periodic + updates on the status of a vnfd + """ + # print(nsd) + # print(vnfd) + print("Workload status: {}".format(workload_status)) + + def test_deploy_multivdu_application(self): + """Deploy a multi-vdu vnf that uses multiple charms.""" + pass + + def test_remove_application(self): + pass + + def test_get_metrics(self): + pass diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..ff6431e --- /dev/null +++ b/tox.ini @@ -0,0 +1,25 @@ +# Tox (http://tox.testrun.org/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + +[tox] +envlist = lint,py35 +skipsdist=True + +[testenv] +basepython=python3 +usedevelop=True +# for testing with other python versions +commands=nosetests +deps = + nose + mock + pyyaml + +[testenv:lint] +envdir = {toxworkdir}/py35 +commands = + flake8 --ignore E501 {posargs} juju tests +deps = + flake8 -- 2.17.1