From: velandy Date: Fri, 31 Mar 2017 15:35:50 +0000 (+0200) Subject: Merge "Revert " Removed unused scaling rpc handler code"" X-Git-Tag: v2.0.0~22 X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=0c8dba8a1e430452e32ea89375ea9f3ee20dc1d0;hp=0fa324c6e4d78e3a681ddf9628d47bf78e4216a3;p=osm%2FSO.git Merge "Revert " Removed unused scaling rpc handler code"" --- diff --git a/charms/layers/.gitignore b/charms/layers/.gitignore new file mode 100644 index 00000000..7da58814 --- /dev/null +++ b/charms/layers/.gitignore @@ -0,0 +1,3 @@ +deps/ +builds/ + diff --git a/charms/layers/sshproxy/.gitignore b/charms/layers/sshproxy/.gitignore new file mode 100644 index 00000000..b8e7ba3a --- /dev/null +++ b/charms/layers/sshproxy/.gitignore @@ -0,0 +1 @@ +trusty/ diff --git a/charms/layers/sshproxy/README.md b/charms/layers/sshproxy/README.md new file mode 100644 index 00000000..c66ff751 --- /dev/null +++ b/charms/layers/sshproxy/README.md @@ -0,0 +1,97 @@ +# Overview + +This is a [Juju] layer intended to ease the development of charms that need +to execute commands over SSH, such as proxy charms. + +# What is a proxy charm? + +A proxy charm is a limited type of charm that does not interact with software running on the same host, such as controlling and configuring a remote device (a static VM image, a router/switch, etc.). It cannot take advantage of some of Juju's key features, such as [scaling], [relations], and [leadership]. + +Proxy charms are primarily a stop-gap, intended to prototype quickly, with the end goal being to develop it into a full-featured charm, which installs and executes code on the same machine as the charm is running. + +# Integration + +After you've [created your charm], open `interfaces.yaml` and add +`layer:sshproxy` to the includes stanza, as shown below: +``` +includes: ['layer:basic', 'layer:sshproxy'] +``` + +## Reactive states + +This layer will set the following states: + +- `sshproxy.configured` This state is set when SSH credentials have been supplied to the charm. + + +## Example +In `reactive/mycharm.py`, you can add logic to execute commands over SSH. This +example is run via a `start` action, and starts a service running on a remote +host. +``` +... +import charms.sshproxy + + +@when('sshproxy.configured') +@when('actions.start') +def start(): + """ Execute's the command, via the start action` using the + configured SSH credentials + """ + sshproxy.ssh("service myservice start") + +``` + +## Actions +This layer includes a built-in `run` action useful for debugging or running arbitrary commands: + +``` +$ juju run-action mycharm/0 run command=hostname +Action queued with id: 014b72f3-bc02-4ecb-8d38-72bce03bbb63 + +$ juju show-action-output 014b72f3-bc02-4ecb-8d38-72bce03bbb63 +results: + output: juju-66a5f3-11 +status: completed +timing: + completed: 2016-10-27 19:53:49 +0000 UTC + enqueued: 2016-10-27 19:53:44 +0000 UTC + started: 2016-10-27 19:53:48 +0000 UTC + +``` +## Known Limitations and Issues + +### Security issues + +- Password and key-based authentications are supported, with the caveat that +both (password and private key) are stored plaintext within the Juju controller. + +# Configuration and Usage + +This layer adds the following configuration options: +- ssh-hostname +- ssh-username +- ssh-password +- ssh-private-key + +Once [configure] those values at any time. Once they are set, the `sshproxy.configured` state flag will be toggled: + +``` +juju deploy mycharm ssh-hostname=10.10.10.10 ssh-username=ubuntu ssh-password=yourpassword +``` +or +``` +juju deploy mycharm ssh-hostname=10.10.10.10 ssh-username=ubuntu ssh-private-key="`cat ~/.ssh/id_rsa`" +``` + + +# Contact Information +Homepage: https://github.com/AdamIsrael/layer-sshproxy + +[Juju]: https://jujucharms.com/about +[configure]: https://jujucharms.com/docs/2.0/charms-config +[scaling]: https://jujucharms.com/docs/2.0/charms-scaling +[relations]: https://jujucharms.com/docs/2.0/charms-relations +[leadership]: https://jujucharms.com/docs/2.0/developer-leadership +[created your charm]: https://jujucharms.com/docs/2.0/developer-getting-started diff --git a/charms/layers/sshproxy/actions.yaml b/charms/layers/sshproxy/actions.yaml new file mode 100644 index 00000000..501b2f99 --- /dev/null +++ b/charms/layers/sshproxy/actions.yaml @@ -0,0 +1,26 @@ +## +# Copyright 2016 Canonical Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + +run: + description: "Run an arbitrary command" + params: + command: + description: "The command to execute." + type: string + default: "" + required: + - command diff --git a/charms/layers/sshproxy/actions/run b/charms/layers/sshproxy/actions/run new file mode 100755 index 00000000..d85d3fac --- /dev/null +++ b/charms/layers/sshproxy/actions/run @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +## +# Copyright 2016 Canonical Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +import sys +sys.path.append('lib') + +from charms.reactive import main +from charms.reactive import set_state +from charmhelpers.core.hookenv import action_fail, action_name + +""" +`set_state` only works here because it's flushed to disk inside the `main()` +loop. remove_state will need to be called inside the action method. +""" +set_state('actions.{}'.format(action_name())) + +try: + main() +except Exception as e: + action_fail(repr(e)) diff --git a/charms/layers/sshproxy/config.yaml b/charms/layers/sshproxy/config.yaml new file mode 100644 index 00000000..07f37568 --- /dev/null +++ b/charms/layers/sshproxy/config.yaml @@ -0,0 +1,34 @@ +## +# Copyright 2016 Canonical Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + +options: + ssh-hostname: + type: string + default: "" + description: "The hostname or IP address of the machine to" + ssh-username: + type: string + default: "" + description: "The username to login as." + ssh-password: + type: string + default: "" + description: "The password used to authenticate." + ssh-private-key: + type: string + default: "" + description: "The private ssh key to be used to authenticate." diff --git a/charms/layers/sshproxy/layer.yaml b/charms/layers/sshproxy/layer.yaml new file mode 100644 index 00000000..4f095a64 --- /dev/null +++ b/charms/layers/sshproxy/layer.yaml @@ -0,0 +1,18 @@ +## +# Copyright 2016 Canonical Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + +includes: ['layer:basic'] # if you use any interfaces, add them here diff --git a/charms/layers/sshproxy/lib/charms/sshproxy.py b/charms/layers/sshproxy/lib/charms/sshproxy.py new file mode 100644 index 00000000..b247bcf2 --- /dev/null +++ b/charms/layers/sshproxy/lib/charms/sshproxy.py @@ -0,0 +1,124 @@ +## +# Copyright 2016 Canonical Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + +from charmhelpers.core.hookenv import ( + config, +) +import io +import paramiko + +from subprocess import ( + Popen, + CalledProcessError, + PIPE, +) + + +def _run(cmd, env=None): + """ Run a command, either on the local machine or remotely via SSH. """ + if isinstance(cmd, str): + cmd = cmd.split(' ') if ' ' in cmd else [cmd] + + cfg = config() + if all(k in cfg for k in ['ssh-hostname', 'ssh-username', + 'ssh-password', 'ssh-private-key']): + host = cfg['ssh-hostname'] + user = cfg['ssh-username'] + passwd = cfg['ssh-password'] + key = cfg['ssh-private-key'] + + if host and user and (passwd or key): + return ssh(cmd, host, user, passwd, key) + + p = Popen(cmd, + env=env, + shell=True, + stdout=PIPE, + stderr=PIPE) + stdout, stderr = p.communicate() + retcode = p.poll() + if retcode > 0: + raise CalledProcessError(returncode=retcode, + cmd=cmd, + output=stderr.decode("utf-8").strip()) + return (stdout.decode('utf-8').strip(), stderr.decode('utf-8').strip()) + + +def get_ssh_client(host, user, password=None, key=None): + """Return a connected Paramiko ssh object""" + + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + pkey = None + if key: + f = io.StringIO(key) + pkey = paramiko.RSAKey.from_private_key(f) + + ########################################################################### + # There is a bug in some versions of OpenSSH 4.3 (CentOS/RHEL 5) where # + # the server may not send the SSH_MSG_USERAUTH_BANNER message except when # + # responding to an auth_none request. For example, paramiko will attempt # + # to use password authentication when a password is set, but the server # + # could deny that, instead requesting keyboard-interactive. The hack to # + # workaround this is to attempt a reconnect, which will receive the right # + # banner, and authentication can proceed. See the following for more info # + # https://github.com/paramiko/paramiko/issues/432 # + # https://github.com/paramiko/paramiko/pull/438 # + ########################################################################### + + try: + client.connect(host, port=22, username=user, + password=password, pkey=pkey) + except paramiko.ssh_exception.SSHException as e: + if 'Error reading SSH protocol banner' == str(e): + # Once more, with feeling + client.connect(host, port=22, username=user, + password=password, pkey=pkey) + pass + else: + raise paramiko.ssh_exception.SSHException(e) + + return client + + +def sftp(local_file, remote_file, host, user, password=None, key=None): + """Copy a local file to a remote host""" + client = get_ssh_client(host, user, password, key) + + # Create an sftp connection from the underlying transport + sftp = paramiko.SFTPClient.from_transport(client.get_transport()) + sftp.put(local_file, remote_file) + client.close() + + +def ssh(cmd, host, user, password=None, key=None): + """ Run an arbitrary command over SSH. """ + client = get_ssh_client(host, user, password, key) + + cmds = ' '.join(cmd) + stdin, stdout, stderr = client.exec_command(cmds, get_pty=True) + retcode = stdout.channel.recv_exit_status() + client.close() # @TODO re-use connections + if retcode > 0: + output = stderr.read().strip() + raise CalledProcessError(returncode=retcode, cmd=cmd, + output=output) + return ( + stdout.read().decode('utf-8').strip(), + stderr.read().decode('utf-8').strip() + ) diff --git a/charms/layers/sshproxy/metadata.yaml b/charms/layers/sshproxy/metadata.yaml new file mode 100644 index 00000000..effeb6a1 --- /dev/null +++ b/charms/layers/sshproxy/metadata.yaml @@ -0,0 +1,31 @@ +## +# Copyright 2016 Canonical Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + +name: sshproxy +summary: Layer to copy files to or run commands on a remote host over ssh +maintainer: Adam Israel +description: | + This layer is intended to provide common ssh functionality, such as + running a command on a remote host. +series: + - trusty + - xenial +tags: + # Replace "misc" with one or more whitelisted tags from this list: + # https://jujucharms.com/docs/stable/authors-charm-metadata + - misc +subordinate: false diff --git a/charms/layers/sshproxy/reactive/sshproxy.py b/charms/layers/sshproxy/reactive/sshproxy.py new file mode 100644 index 00000000..e1508ca0 --- /dev/null +++ b/charms/layers/sshproxy/reactive/sshproxy.py @@ -0,0 +1,73 @@ +## +# Copyright 2016 Canonical Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + +from charmhelpers.core.hookenv import ( + action_fail, + action_get, + action_set, + config, +) +from charms.reactive import ( + remove_state, + set_state, + when, +) +import charms.sshproxy +import subprocess + + +@when('config.changed') +def ssh_configured(): + """ Checks to see if the charm is configured with SSH credentials. If so, + set a state flag that can be used to execute ssh-only actions. + + For example: + + @when('sshproxy.configured') + def run_remote_command(cmd): + ... + + @when_not('sshproxy.configured') + def run_local_command(cmd): + ... + """ + cfg = config() + if all(k in cfg for k in ['ssh-hostname', 'ssh-username', + 'ssh-password', 'ssh-private-key']): + set_state('sshproxy.configured') + else: + remove_state('sshproxy.configured') + + +@when('actions.run') +def run_command(): + """ + Run an arbitrary command, either locally or over SSH with the configured + credentials. + """ + try: + cmd = action_get('command') + output, err = charms.sshproxy._run(cmd) + if len(err): + action_fail("Command '{}' returned error code {}".format(cmd, err)) + else: + action_set({'output': output}) + except subprocess.CalledProcessError as e: + action_fail('Command failed: %s (%s)' % + (' '.join(e.cmd), str(e.output))) + finally: + remove_state('actions.run') diff --git a/charms/layers/sshproxy/tests/00-setup b/charms/layers/sshproxy/tests/00-setup new file mode 100755 index 00000000..1e5f91bc --- /dev/null +++ b/charms/layers/sshproxy/tests/00-setup @@ -0,0 +1,22 @@ +## +# Copyright 2016 Canonical Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + +#!/bin/bash + +sudo add-apt-repository ppa:juju/stable -y +sudo apt-get update +sudo apt-get install amulet python-requests -y diff --git a/charms/layers/sshproxy/tests/10-deploy b/charms/layers/sshproxy/tests/10-deploy new file mode 100755 index 00000000..f5d2f5d6 --- /dev/null +++ b/charms/layers/sshproxy/tests/10-deploy @@ -0,0 +1,76 @@ +## +# Copyright 2016 Canonical Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + +#!/usr/bin/python3 + +import amulet +import requests +import unittest +import string +import random + + +class TestCharm(unittest.TestCase): + user = None + passwd = None + + def id_generator(self, size=6, chars=string.ascii_uppercase + string.digits): + return ''.join(random.choice(chars) for _ in range(size)) + + def setUp(self): + + # Setup random user/password + self.user = self.id_generator() + self.passwd = self.id_generator() + + self.d = amulet.Deployment() + + self.d.add('sshproxy') + self.d.add('ubuntu') + + self.d.expose('sshproxy') + + self.d.setup(timeout=900) + self.d.sentry.wait() + + # Add + ubuntu_0 = d.sentry['ubuntu'][0] + ubuntu_0.ssh("sudo adduser {}".format(self.user)) + ubuntu_0.ssh("echo '{}' | sudo passwd {} --stdin".format(self.passwd, self.user)) + + self.unit = self.d.sentry['sshproxy'][0] + + def test_service(self): + + # Configure the unit + + # Run a command + + # Verify the output + + # test we can access over http + # page = requests.get('http://{}'.format(self.unit.info['public-address'])) + # self.assertEqual(page.status_code, 200) + # Now you can use self.d.sentry[SERVICE][UNIT] to address each of the units and perform + # more in-depth steps. Each self.d.sentry[SERVICE][UNIT] has the following methods: + # - .info - An array of the information of that unit from Juju + # - .file(PATH) - Get the details of a file on that unit + # - .file_contents(PATH) - Get plain text output of PATH file from that unit + # - .directory(PATH) - Get details of directory + # - .directory_contents(PATH) - List files and folders in PATH on that unit + # - .relation(relation, service:rel) - Get relation data from return service + pass diff --git a/charms/layers/sshproxy/wheelhouse.txt b/charms/layers/sshproxy/wheelhouse.txt new file mode 100644 index 00000000..ab6de07b --- /dev/null +++ b/charms/layers/sshproxy/wheelhouse.txt @@ -0,0 +1,18 @@ +## +# Copyright 2016 Canonical Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + +paramiko>=1.16.0,<1.17 diff --git a/charms/layers/vnfproxy/README.md b/charms/layers/vnfproxy/README.md new file mode 100644 index 00000000..8379e50c --- /dev/null +++ b/charms/layers/vnfproxy/README.md @@ -0,0 +1,172 @@ +# vnfproxy + +## Overview + +This charm layer is intended for use by vendors who wish to integrate with +OSM. The current release of OSM only supports a lightweight version of Juju +charms, which we will refer to as "proxy charms". Consider the diagram below: + +``` ++---------------------+ +---------------------+ +| <----+ | +| Resource | | Service | +| Orchestrator (RO) +----> Orchestrator (SO) | +| | | | ++------------------+--+ +-------+----^--------+ + | | | + | | | + | | | + +-----v-----+ +-v----+--+ + | <-------+ | + | Virtual | | Proxy | + | Machine | | Charm | + | +-------> | + +-----------+ +---------+ +``` + +The Virtual Machine (VM) is created by the Resource Orchestrator (RO), at the +request of the Service Orchestrator (SO). Once the VM has been created, a +"proxy charm" is deployed in order to facilitate operations between the SO and +your service running within the VM. + +As such, a proxy charm will expose a number of "actions" that are run via the +SO. By default, the following actions are exposed: + +```bash +actions +├── reboot +├── restart +├── run +├── start +└── stop +``` + +Some actions, such as `run` and `reboot`, do not require any additional configuration. `start`, `stop` and `restart`, however, will require you to +implement the command(s) required to interact with your service. + +## Usage + +Create the framework for your proxy charm: + +```bash +$ charm create pingpong +$ cd pingpong +``` + +Modify `layer.yaml` to the following: +```yaml +includes: + - layer:basic + - layer:vnfproxy +``` + +The `metadata.yaml` describes your service. It should look similar to the following: + +```yaml +name: vnfproxy +summary: A layer for developing OSM "proxy" charms. +maintainer: Adam Israel +description: | + VNF "proxy" charms are a lightweight version of a charm that, rather than + installing software on the same machine, execute commands over an ssh channel. +series: + - trusty + - xenial +tags: + - osm + - vnf +subordinate: false +``` + +Implement the default action(s) you wish to support by adding the following code to reactive/pingpong.py and fill in the cmd to be run: + +```python +@when('actions.start') +def start(): + err = '' + try: + cmd = "" + result, err = charms.sshproxy._run(cmd) + except: + action_fail('command failed:' + err) + else: + action_set({'outout': result}) + finally: + remove_flag('actions.start') + + +@when('actions.stop') +def stop(): + err = '' + try: + # Enter the command to stop your service(s) + cmd = "service myname stop" + result, err = charms.sshproxy._run(cmd) + except: + action_fail('command failed:' + err) + else: + action_set({'outout': result}) + finally: + remove_flag('actions.stop') + + +@when('actions.restart') +def restart(): + err = '' + try: + # Enter the command to restart your service(s) + cmd = "service myname restart" + result, err = charms.sshproxy._run(cmd) + except: + action_fail('command failed:' + err) + else: + action_set({'outout': result}) + finally: + remove_flag('actions.restart') +``` + +Rename `README.ex` to `README.md` and describe your application and its usage. + +-- fix this. there are cases where the config is useful -- Delete `config.yaml`, since the charm's configuration will be driven by the SO. + +Create the `actions.yaml` file; this will describe the additional operations you would like to perform on or against your service. + +```yaml +set-server: + description: "Set the target IP address and port" + params: + server-ip: + description: "IP on which the target service is listening." + type: string + default: "" + server-port: + description: "Port on which the target service is listening." + type: integer + default: 5555 + required: + - server-ip +set-rate: + description: "Set the rate of packet generation." + params: + rate: + description: "Packet rate." + type: integer + default: 5 +get-stats: + description: "Get the stats." +get-state: + description: "Get the admin state of the target service." +get-rate: + description: "Get the rate set on the target service." +get-server: + description: "Get the target server and IP set" +``` + + +Once you've implemented your actions, you need to compile the various charm layers: +```bash +$ charm build + +``` + +## Contact diff --git a/charms/layers/vnfproxy/actions.yaml b/charms/layers/vnfproxy/actions.yaml new file mode 100644 index 00000000..a9ee2ba0 --- /dev/null +++ b/charms/layers/vnfproxy/actions.yaml @@ -0,0 +1,25 @@ +## +# Copyright 2016 Canonical Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + +start: + description: "Stop the service" +stop: + description: "Stop the service" +restart: + description: "Stop the service" +reboot: + description: "Reboot the machine" diff --git a/charms/layers/vnfproxy/actions/reboot b/charms/layers/vnfproxy/actions/reboot new file mode 100755 index 00000000..751c88cf --- /dev/null +++ b/charms/layers/vnfproxy/actions/reboot @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +## +# Copyright 2016 Canonical Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +import sys +sys.path.append('lib') + +from charms.reactive import main +from charms.reactive import set_state +from charmhelpers.core.hookenv import action_fail + +""" +`set_state` only works here because it's flushed to disk inside the `main()` +loop. remove_state will need to be called inside the action method. +""" +set_state('actions.reboot') + +try: + main() +except Exception as e: + action_fail(repr(e)) diff --git a/charms/layers/vnfproxy/actions/restart b/charms/layers/vnfproxy/actions/restart new file mode 100755 index 00000000..f62929b1 --- /dev/null +++ b/charms/layers/vnfproxy/actions/restart @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +## +# Copyright 2016 Canonical Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + +import sys +sys.path.append('lib') + +from charms.reactive import main +from charms.reactive import set_state +from charmhelpers.core.hookenv import action_fail + +""" +`set_state` only works here because it's flushed to disk inside the `main()` +loop. remove_state will need to be called inside the action method. +""" +set_state('actions.restart') + +try: + main() +except Exception as e: + action_fail(repr(e)) diff --git a/charms/layers/vnfproxy/actions/start b/charms/layers/vnfproxy/actions/start new file mode 100755 index 00000000..a9db3d07 --- /dev/null +++ b/charms/layers/vnfproxy/actions/start @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +## +# Copyright 2016 Canonical Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + +import sys +sys.path.append('lib') + +from charms.reactive import main +from charms.reactive import set_state +from charmhelpers.core.hookenv import action_fail + +""" +`set_state` only works here because it's flushed to disk inside the `main()` +loop. remove_state will need to be called inside the action method. +""" +set_state('actions.start') + +try: + main() +except Exception as e: + action_fail(repr(e)) diff --git a/charms/layers/vnfproxy/actions/stop b/charms/layers/vnfproxy/actions/stop new file mode 100755 index 00000000..38904868 --- /dev/null +++ b/charms/layers/vnfproxy/actions/stop @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +## +# Copyright 2016 Canonical Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + +import sys +sys.path.append('lib') + +from charms.reactive import main +from charms.reactive import set_state +from charmhelpers.core.hookenv import action_fail + +""" +`set_state` only works here because it's flushed to disk inside the `main()` +loop. remove_state will need to be called inside the action method. +""" +set_state('actions.stop') + +try: + main() +except Exception as e: + action_fail(repr(e)) diff --git a/charms/layers/vnfproxy/icon.svg b/charms/layers/vnfproxy/icon.svg new file mode 100644 index 00000000..e092eef7 --- /dev/null +++ b/charms/layers/vnfproxy/icon.svg @@ -0,0 +1,279 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + diff --git a/charms/layers/vnfproxy/layer.yaml b/charms/layers/vnfproxy/layer.yaml new file mode 100644 index 00000000..89bf2457 --- /dev/null +++ b/charms/layers/vnfproxy/layer.yaml @@ -0,0 +1,20 @@ +## +# Copyright 2016 Canonical Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + +includes: + - layer:basic + - layer:sshproxy diff --git a/charms/layers/vnfproxy/metadata.yaml b/charms/layers/vnfproxy/metadata.yaml new file mode 100644 index 00000000..909643d5 --- /dev/null +++ b/charms/layers/vnfproxy/metadata.yaml @@ -0,0 +1,30 @@ +## +# Copyright 2016 Canonical Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + +name: vnfproxy +summary: A layer for developing OSM "proxy" charms. +maintainer: Adam Israel +description: | + VNF "proxy" charms are a lightweight version of a charm that, rather than + installing software on the same machine, execute commands over an ssh channel. +series: + - trusty + - xenial +tags: + - osm + - vnf +subordinate: false diff --git a/charms/layers/vnfproxy/reactive/vnfproxy.py b/charms/layers/vnfproxy/reactive/vnfproxy.py new file mode 100644 index 00000000..2b26212b --- /dev/null +++ b/charms/layers/vnfproxy/reactive/vnfproxy.py @@ -0,0 +1,75 @@ +from charmhelpers.core.hookenv import ( + action_fail, + action_set, +) + +from charms.reactive import ( + when, + remove_state as remove_flag, +) +import charms.sshproxy + + +@when('actions.reboot') +def reboot(): + err = '' + try: + result, err = charms.sshproxy._run("reboot") + except: + action_fail('command failed:' + err) + else: + action_set({'outout': result}) + finally: + remove_flag('actions.reboot') + + +############################################################################### +# Below is an example implementation of the start/stop/restart actions. # +# To use this, copy the below code into your layer and add the appropriate # +# command(s) necessary to perform the action. # +############################################################################### + +# @when('actions.start') +# def start(): +# err = '' +# try: +# cmd = "service myname start" +# result, err = charms.sshproxy._run(cmd) +# except: +# action_fail('command failed:' + err) +# else: +# action_set({'outout': result}) +# finally: +# remove_flag('actions.start') +# +# +# @when('actions.stop') +# def stop(): +# err = '' +# try: +# # Enter the command to stop your service(s) +# cmd = "service myname stop" +# result, err = charms.sshproxy._run(cmd) +# except: +# action_fail('command failed:' + err) +# else: +# action_set({'outout': result}) +# finally: +# remove_flag('actions.stop') +# +# +# @when('actions.restart') +# def restart(): +# err = '' +# try: +# # Enter the command to restart your service(s) +# cmd = "service myname restart" +# result, err = charms.sshproxy._run(cmd) +# except: +# action_fail('command failed:' + err) +# else: +# action_set({'outout': result}) +# finally: +# remove_flag('actions.restart') +# +# diff --git a/charms/layers/vnfproxy/tests/00-setup b/charms/layers/vnfproxy/tests/00-setup new file mode 100755 index 00000000..1e5f91bc --- /dev/null +++ b/charms/layers/vnfproxy/tests/00-setup @@ -0,0 +1,22 @@ +## +# Copyright 2016 Canonical Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + +#!/bin/bash + +sudo add-apt-repository ppa:juju/stable -y +sudo apt-get update +sudo apt-get install amulet python-requests -y diff --git a/charms/layers/vnfproxy/tests/10-deploy b/charms/layers/vnfproxy/tests/10-deploy new file mode 100755 index 00000000..5bb90445 --- /dev/null +++ b/charms/layers/vnfproxy/tests/10-deploy @@ -0,0 +1,52 @@ +## +# Copyright 2016 Canonical Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + +#!/usr/bin/python3 + +import amulet +import requests +import unittest + + +class TestCharm(unittest.TestCase): + def setUp(self): + self.d = amulet.Deployment() + + self.d.add('vnfproxy') + self.d.expose('vnfproxy') + + self.d.setup(timeout=900) + self.d.sentry.wait() + + self.unit = self.d.sentry['vnfproxy'][0] + + def test_service(self): + # test we can access over http + page = requests.get('http://{}'.format(self.unit.info['public-address'])) + self.assertEqual(page.status_code, 200) + # Now you can use self.d.sentry[SERVICE][UNIT] to address each of the units and perform + # more in-depth steps. Each self.d.sentry[SERVICE][UNIT] has the following methods: + # - .info - An array of the information of that unit from Juju + # - .file(PATH) - Get the details of a file on that unit + # - .file_contents(PATH) - Get plain text output of PATH file from that unit + # - .directory(PATH) - Get details of directory + # - .directory_contents(PATH) - List files and folders in PATH on that unit + # - .relation(relation, service:rel) - Get relation data from return service + + +if __name__ == '__main__': + unittest.main() diff --git a/rwlaunchpad/plugins/rwimagemgr/rift/tasklets/rwimagemgr/upload.py b/rwlaunchpad/plugins/rwimagemgr/rift/tasklets/rwimagemgr/upload.py index 7ce74b22..c1716d3c 100644 --- a/rwlaunchpad/plugins/rwimagemgr/rift/tasklets/rwimagemgr/upload.py +++ b/rwlaunchpad/plugins/rwimagemgr/rift/tasklets/rwimagemgr/upload.py @@ -250,6 +250,8 @@ class ImageUploadJob(object): if failed_tasks: self._log.error("%s had %s FAILED tasks.", self, len(failed_tasks)) + for ftask in failed_tasks: + self._log.error("%s : Failed to upload image : %s to cloud_account : %s", self, ftask.image_name, ftask.cloud_account) self.state = "FAILED" else: self._log.debug("%s tasks completed successfully", len(self._upload_tasks)) diff --git a/rwlaunchpad/plugins/rwlaunchpadtasklet/rift/tasklets/rwlaunchpad/image.py b/rwlaunchpad/plugins/rwlaunchpadtasklet/rift/tasklets/rwlaunchpad/image.py index b18e304d..7c4dfa07 100644 --- a/rwlaunchpad/plugins/rwlaunchpadtasklet/rift/tasklets/rwlaunchpad/image.py +++ b/rwlaunchpad/plugins/rwlaunchpadtasklet/rift/tasklets/rwlaunchpad/image.py @@ -71,4 +71,5 @@ class ImageUploader(object): try: upload_job.wait_until_complete_threadsafe() except client.UploadJobError as e: - raise ImageUploadError("Failed to upload image (image_name) to cloud accounts") from e + raise ImageUploadError("Failed to upload image " + image_name + " to cloud accounts") from e + diff --git a/rwlaunchpad/plugins/rwlaunchpadtasklet/scripts/onboard_pkg b/rwlaunchpad/plugins/rwlaunchpadtasklet/scripts/onboard_pkg index 2b2eb911..ba82e7ed 100755 --- a/rwlaunchpad/plugins/rwlaunchpadtasklet/scripts/onboard_pkg +++ b/rwlaunchpad/plugins/rwlaunchpadtasklet/scripts/onboard_pkg @@ -375,14 +375,24 @@ class OnboardPackage: msg = "Error instantiating NS as {} with NSD {}: ". \ format(self._service_name, self._nsd_id, reply["rpc-error"]) - self.log.error(msg) + # self.log.error(msg) raise OnboardPkgInstError(msg) self.log.info("Successfully initiated instantiation of NS as {} ({})". format(self._service_name, ns_id)) def process(self): - self.validate_args() + try: + self.validate_args() + except Exception as e: + if args.verbose: + log.exception(e) + + print("\nERROR:", e) + print("\n") + parser.print_help() + sys.exit(2) + self.validate_connectivity() self.upload_packages() self.instantiate() @@ -425,15 +435,24 @@ if __name__ == "__main__": fmt = logging.Formatter( '%(asctime)-23s %(levelname)-5s (%(name)s@%(process)d:' \ '%(filename)s:%(lineno)d) - %(message)s') - stderr_handler = logging.StreamHandler(stream=sys.stderr) - stderr_handler.setFormatter(fmt) - logging.basicConfig(level=logging.INFO) log = logging.getLogger('onboard-pkg') - log.addHandler(stderr_handler) + log.setLevel(logging.INFO) if args.verbose: log.setLevel(logging.DEBUG) + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + ch.setFormatter(fmt) + log.addHandler(ch) log.debug("Input arguments: {}".format(args)) - ob = OnboardPackage(log, args) - ob.process() + try: + ob = OnboardPackage(log, args) + ob.process() + except Exception as e: + if args.verbose: + log.exception(e) + + print("\nERROR:", e) + sys.exit(1) + diff --git a/rwlaunchpad/plugins/rwpkgmgr/rift/tasklets/rwpkgmgr/publisher/download_status.py b/rwlaunchpad/plugins/rwpkgmgr/rift/tasklets/rwpkgmgr/publisher/download_status.py index 6890241e..d8c6ade5 100644 --- a/rwlaunchpad/plugins/rwpkgmgr/rift/tasklets/rwpkgmgr/publisher/download_status.py +++ b/rwlaunchpad/plugins/rwpkgmgr/rift/tasklets/rwpkgmgr/publisher/download_status.py @@ -18,13 +18,18 @@ # import asyncio -import uuid +import sys from gi.repository import (RwDts as rwdts) import rift.mano.dts as mano_dts import rift.downloader as url_downloader +import functools +import concurrent + +if sys.version_info < (3, 4, 4): + asyncio.ensure_future = asyncio.async class DownloadStatusPublisher(mano_dts.DtsHandler, url_downloader.DownloaderProtocol): @@ -32,26 +37,53 @@ class DownloadStatusPublisher(mano_dts.DtsHandler, url_downloader.DownloaderProt super().__init__(log, dts, loop) self.tasks = {} + def xpath(self, download_id=None): return ("D,/rw-pkg-mgmt:download-jobs/rw-pkg-mgmt:job" + ("[download-id='{}']".format(download_id) if download_id else "")) + @asyncio.coroutine + def _dts_publisher(self, job): + # Publish the download state + self.reg.update_element( + self.xpath(download_id=job.download_id), job) + @asyncio.coroutine def register(self): self.reg = yield from self.dts.register(xpath=self.xpath(), flags=rwdts.Flag.PUBLISHER|rwdts.Flag.CACHE|rwdts.Flag.NO_PREP_READ) assert self.reg is not None - + + @staticmethod + def _async_func(func, fut): + try: + ret = func() + fut.set_result(ret) + except Exception as e: + fut.set_exception(e) + + def _schedule_dts_work(self, download_job_msg): + # Create a coroutine + cort = self._dts_publisher(download_job_msg) + # Use main asyncio loop (running in main thread) + newfunc = functools.partial(asyncio.ensure_future, cort, loop=self.loop) + fut = concurrent.futures.Future() + # Schedule future in main thread immediately + self.loop.call_soon_threadsafe(DownloadStatusPublisher._async_func, newfunc, fut) + res = fut.result() + exc = fut.exception() + if exc is not None: + self.log.error("Caught future exception during download: %s type %s", str(exc), type(exc)) + raise exc + return res def on_download_progress(self, download_job_msg): """callback that triggers update. """ - key = download_job_msg.download_id # Trigger progess update - self.reg.update_element( - self.xpath(download_id=key), - download_job_msg) + # Schedule a future in the main thread + self._schedule_dts_work(download_job_msg) def on_download_finished(self, download_job_msg): """callback that triggers update. @@ -63,9 +95,8 @@ class DownloadStatusPublisher(mano_dts.DtsHandler, url_downloader.DownloaderProt del self.tasks[key] # Publish the final state - self.reg.update_element( - self.xpath(download_id=key), - download_job_msg) + # Schedule a future in the main thread + self._schedule_dts_work(download_job_msg) @asyncio.coroutine def register_downloader(self, downloader): diff --git a/rwlaunchpad/plugins/rwpkgmgr/test/utest_publisher_dts.py b/rwlaunchpad/plugins/rwpkgmgr/test/utest_publisher_dts.py index 9ec29561..a02e5c66 100755 --- a/rwlaunchpad/plugins/rwpkgmgr/test/utest_publisher_dts.py +++ b/rwlaunchpad/plugins/rwpkgmgr/test/utest_publisher_dts.py @@ -99,7 +99,7 @@ class TestCase(rift.test.dts.AbstractDTSTest): "package_id": "123", "download_id": str(uuid.uuid4())}) - self.job_handler.on_download_progress(mock_msg) + yield from self.job_handler._dts_publisher(mock_msg) yield from asyncio.sleep(5, loop=self.loop) itr = yield from self.dts.query_read("/download-jobs/job[download-id='{}']".format( @@ -110,12 +110,12 @@ class TestCase(rift.test.dts.AbstractDTSTest): result = yield from fut result = result.result - print (mock_msg) + print ("Mock ", mock_msg) assert result == mock_msg # Modify the msg mock_msg.url = "http://bar/foo" - self.job_handler.on_download_finished(mock_msg) + yield from self.job_handler._dts_publisher(mock_msg) yield from asyncio.sleep(5, loop=self.loop) itr = yield from self.dts.query_read("/download-jobs/job[download-id='{}']".format( @@ -138,17 +138,18 @@ class TestCase(rift.test.dts.AbstractDTSTest): proxy = mock.MagicMock() - url = "http://boson.eng.riftio.com/common/unittests/rift-shell" + url = "http://boson.eng.riftio.com/common/unittests/plantuml.jar" url_downloader = downloader.PackageFileDownloader(url, "1", "/", "VNFD", proxy) download_id = yield from self.job_handler.register_downloader(url_downloader) assert download_id is not None - + + # Waiting for 5 secs to be sure that the file is downloaded yield from asyncio.sleep(5, loop=self.loop) xpath = "/download-jobs/job[download-id='{}']".format( download_id) result = yield from self.read_xpath(xpath) - print (result) + self.log.debug("Test result before complete check - %s", result) assert result.status == "COMPLETED" assert len(self.job_handler.tasks) == 0 @@ -171,14 +172,16 @@ class TestCase(rift.test.dts.AbstractDTSTest): xpath = "/download-jobs/job[download-id='{}']".format( download_id) - yield from asyncio.sleep(3, loop=self.loop) + yield from asyncio.sleep(1, loop=self.loop) result = yield from self.read_xpath(xpath) + self.log.debug("Test result before in_progress check - %s", result) assert result.status == "IN_PROGRESS" yield from self.job_handler.cancel_download(download_id) yield from asyncio.sleep(3, loop=self.loop) result = yield from self.read_xpath(xpath) + self.log.debug("Test result before cancel check - %s", result) assert result.status == "CANCELLED" assert len(self.job_handler.tasks) == 0