Squashed 'modules/libjuju/' changes from c50c361..c127833
c127833 Bump version and changelog for release
6aff679 k8s bundles no longer have application placement (#293)
1de9ad1 Add retry for connection if all endpoints fail (#288)
8cb8d75 Support generation of registration string for model sharing. (#279)
a9e2fd6 Add Twine for dist upload on release (#284)
407a6a6 Update and prepare for 0.11.2 release (#282)
d102620 call related update credential cloud facade methods based on facade version (#281)
2acbdc4 Add test case for redirect during connect (#275)
35fb43e Implement App.get_resources and pinned resources in bundles (#278)
b5ba51a Bump version and changelog for release
7a73a0a Fix bundles with subordinates for Juju <2.5 (#277)
a0f950f Bump version and changelog for release
01125e2 Updates for new Juju version (#274)
87d9388 Fix wrong variable name in revoke_model function (#271)
2b43065 Bump version and changelog for release
98ee524 set include_stats to false to reduce request time (#266)
61e1d69 Update version and changelog for 0.10.1
82f9968 Retry ssh in manual provision test (#265)
d64bfff Clean up lint and add lint coverage to travis config (#263)
c7c5c54 Increase the timeout for charmstore connections (#262)
4a6e398 Fix log level of `Driver connected to juju` message (#258)
514e479 Update version and changelog for 0.10.0
ec2c493 Reorder scp parameters (#259) (#260)
26c86c8 Implement set/get model constraints (#253)
c6b4ab4 Update version and changelog for 0.9.1
e863746 Update websockets to 6.0 (#254)
567bc1a Update version and changelog for 0.9.0
b275ced python3.7 compatibility updates (#251)
bc7336a Handle juju not installed in is_bootstrapped. (#250)
1ce8e0b Add app.reset_config(list). (#249)
c620d4f Implement model.get_action_status (#248)
96ea3c4 Fix `make client` in Python 3.6 (#247)
61969ea Update version and changelog for release
ebf6882 Add support for adding a manual (ssh) machine (#240)
18422f4 Backwards compatibility fixes (#213)
40c0211 Implement model.get_action_output (#242)
c6b8ac5 Fix JSON serialization error for bundle with lxd to unit placement (#243)
5014fc3 Fix reference in docs to connect_current (#239)
ebe0193 Wrap machine agent status workaround in version check (#238)
462989b Convert seconds to nanoseconds for juju.unit.run (#237)
0f413e6 Fix spurious intermittent failure in test_machines.py::test_status (#236)
ce36b60 Define an unused juju-zfs lxd storage pool (#235)
dfc2e8d Add support for Application get_actions (#234)
e7e8c13 Update version and changelog for release
499337b Surface errors from bundle plan (#233)
2d94186 Always send auth-tag even with macaroon auth (#217)
000355c Inline jsonfile credential when sending to controller (#231)
9805123 Bump VERSION and changelog for release
27d723b Always parse tags and spaces constraints to lists (#228)
668945a Doc index improvements (#211)
65e6b5e Add doc req to force newer pymacaroons to fix RTD builds
e2abd47 Fix dependency conflict for building docs
2907a6e Bump VERSION and changelog for 0.7.3 release
37a7500 Full macaroon bakery support (#206)
a06e313 Fix regression with deploying local charm, add test case (#209)
75e9a2b Expose a machines series (#208)
46c98f5 Revert non-functional switch to Py3.6, just specify Py3 instead (#205)
8a99ad1 Cherry-pick VERSION and changelog bump from 0.7.2 release branch
88121d6 Support deploying bundle YAML file directly (rather than just directory) (#202)
57c0dbf Cherry-pick #197 into master (#198)
0973edc Update VERSION and changelog for 0.7.0
f5a4108 Add deprecated placeholder for Controller.get_models
17dffa4 JujuData abstract base class (#194)
76f22cc Make Model and Controller connect methods backwardly compatible (#196)
19b5658 Fix race condition in adding relations (#192)
978f35c refactor connections prior to bakery authentication (#187)
77c0f04 sort all imports; lint tests (#188)
4740935 juju.client.gocookies: new module (#186)
2c4de22 all: use pyrfc3339 instead of dateutil (#185)
7133ffe juju/client: factor out JujuData class (#182)
476b832 Fix race condition in connection monitor test (#183)
e64a5d1 Fix example in README (#178)
97355cc Fix rare hang during Unit.run (#177)
ae0b091 #176: Fix licensing quirks
c0d001b Refactor model handling (#171)
ab807c8 Refactor users handling, add get_users (#170)
5270db5 Upload credential to controller when adding model (#168)
16d8390 Support 'applications' key in bundles (#165)
2de3eed Improve handling of thread error handling for loop.run() (#169)
7807023 Fix encoding when using to_json() (#166)
73effb1 Fix intermittent test failures (#167)
46da148 Update VERSION and changelog for release
3dda1dc Fix test failures (#163)
14392af removing cli command to add ssh keys (#161)
ce68170 Make Application.upgrade_charm upgrade resources (#158)
git-subtree-dir: modules/libjuju
git-subtree-split: c12783304945fdff5c28397b82b535a9cc065ca3
diff --git a/juju/provisioner.py b/juju/provisioner.py
new file mode 100644
index 0000000..da8be16
--- /dev/null
+++ b/juju/provisioner.py
@@ -0,0 +1,366 @@
+from .client import client
+
+import paramiko
+import os
+import re
+import tempfile
+import shlex
+from subprocess import (
+ CalledProcessError,
+)
+import uuid
+
+
+arches = [
+ [re.compile(r"amd64|x86_64"), "amd64"],
+ [re.compile(r"i?[3-9]86"), "i386"],
+ [re.compile(r"(arm$)|(armv.*)"), "armhf"],
+ [re.compile(r"aarch64"), "arm64"],
+ [re.compile(r"ppc64|ppc64el|ppc64le"), "ppc64el"],
+ [re.compile(r"s390x?"), "s390x"],
+
+]
+
+
+def normalize_arch(rawArch):
+ """Normalize the architecture string."""
+ for arch in arches:
+ if arch[0].match(rawArch):
+ return arch[1]
+
+
+DETECTION_SCRIPT = """#!/bin/bash
+set -e
+os_id=$(grep '^ID=' /etc/os-release | tr -d '"' | cut -d= -f2)
+if [ "$os_id" = 'centos' ]; then
+ os_version=$(grep '^VERSION_ID=' /etc/os-release | tr -d '"' | cut -d= -f2)
+ echo "centos$os_version"
+else
+ lsb_release -cs
+fi
+uname -m
+grep MemTotal /proc/meminfo
+cat /proc/cpuinfo
+"""
+
+INITIALIZE_UBUNTU_SCRIPT = """set -e
+(id ubuntu &> /dev/null) || useradd -m ubuntu -s /bin/bash
+umask 0077
+temp=$(mktemp)
+echo 'ubuntu ALL=(ALL) NOPASSWD:ALL' > $temp
+install -m 0440 $temp /etc/sudoers.d/90-juju-ubuntu
+rm $temp
+su ubuntu -c 'install -D -m 0600 /dev/null ~/.ssh/authorized_keys'
+export authorized_keys="{}"
+if [ ! -z "$authorized_keys" ]; then
+ su ubuntu -c 'echo $authorized_keys >> ~/.ssh/authorized_keys'
+fi
+"""
+
+
+class SSHProvisioner:
+ """Provision a manually created machine via SSH."""
+ user = ""
+ host = ""
+ private_key_path = ""
+
+ def __init__(self, user, host, private_key_path):
+ self.host = host
+ self.user = user
+ self.private_key_path = private_key_path
+
+ def _get_ssh_client(self, host, user, key):
+ """Return a connected Paramiko ssh object.
+
+ :param str host: The host to connect to.
+ :param str user: The user to connect as.
+ :param str key: The private key to authenticate with.
+
+ :return: object: A paramiko.SSHClient
+ :raises: :class:`paramiko.ssh_exception.SSHException` if the
+ connection failed
+ """
+
+ ssh = paramiko.SSHClient()
+ ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+ pkey = None
+
+ # Read the private key into a paramiko.RSAKey
+ if os.path.exists(key):
+ with open(key, 'r') as f:
+ pkey = paramiko.RSAKey.from_private_key(f)
+
+ #######################################################################
+ # There is a bug in some versions of OpenSSH 4.3 (CentOS/RHEL5) where #
+ # the server may not send the SSH_MSG_USERAUTH_BANNER message except #
+ # when responding to an auth_none request. For example, paramiko will #
+ # attempt to use password authentication when a password is set, but #
+ # the server could deny that, instead requesting keyboard-interactive.#
+ # The hack to workaround this is to attempt a reconnect, which will #
+ # receive the right banner, and authentication can proceed. See the #
+ # following for more info: #
+ # https://github.com/paramiko/paramiko/issues/432 #
+ # https://github.com/paramiko/paramiko/pull/438 #
+ #######################################################################
+
+ try:
+ ssh.connect(host, port=22, username=user, pkey=pkey)
+ except paramiko.ssh_exception.SSHException as e:
+ if 'Error reading SSH protocol banner' == str(e):
+ # Once more, with feeling
+ ssh.connect(host, port=22, username=user, pkey=pkey)
+ else:
+ # Reraise the original exception
+ raise e
+
+ return ssh
+
+ def _run_command(self, ssh, cmd, pty=True):
+ """Run a command remotely via SSH.
+
+ :param object ssh: The SSHClient
+ :param str cmd: The command to execute
+ :param list cmd: The `shlex.split` command to execute
+ :param bool pty: Whether to allocate a pty
+
+ :return: tuple: The stdout and stderr of the command execution
+ :raises: :class:`CalledProcessError` if the command fails
+ """
+
+ if isinstance(cmd, str):
+ cmd = shlex.split(cmd)
+
+ if type(cmd) is not list:
+ cmd = [cmd]
+
+ cmds = ' '.join(cmd)
+ stdin, stdout, stderr = ssh.exec_command(cmds, get_pty=pty)
+ retcode = stdout.channel.recv_exit_status()
+
+ if retcode > 0:
+ output = stderr.read().strip()
+ raise CalledProcessError(returncode=retcode, cmd=cmd,
+ output=output)
+ return (
+ stdout.read().decode('utf-8').strip(),
+ stderr.read().decode('utf-8').strip()
+ )
+
+ def _init_ubuntu_user(self):
+ """Initialize the ubuntu user.
+
+ :return: bool: If the initialization was successful
+ :raises: :class:`paramiko.ssh_exception.AuthenticationException`
+ if the authentication fails
+ """
+
+ # TODO: Test this on an image without the ubuntu user setup.
+
+ auth_user = self.user
+ ssh = None
+ try:
+ # Run w/o allocating a pty, so we fail if sudo prompts for a passwd
+ ssh = self._get_ssh_client(
+ self.host,
+ "ubuntu",
+ self.private_key_path,
+ )
+
+ stdout, stderr = self._run_command(ssh, "sudo -n true", pty=False)
+ except paramiko.ssh_exception.AuthenticationException as e:
+ raise e
+ else:
+ auth_user = "ubuntu"
+ finally:
+ if ssh:
+ ssh.close()
+
+ # if the above fails, run the init script as the authenticated user
+
+ # Infer the public key
+ public_key = None
+ public_key_path = "{}.pub".format(self.private_key_path)
+
+ if not os.path.exists(public_key_path):
+ raise FileNotFoundError(
+ "Public key '{}' doesn't exist.".format(public_key_path)
+ )
+
+ with open(public_key_path, "r") as f:
+ public_key = f.readline()
+
+ script = INITIALIZE_UBUNTU_SCRIPT.format(public_key)
+
+ try:
+ ssh = self._get_ssh_client(
+ self.host,
+ auth_user,
+ self.private_key_path,
+ )
+
+ self._run_command(
+ ssh,
+ ["sudo", "/bin/bash -c " + shlex.quote(script)],
+ pty=True
+ )
+ except paramiko.ssh_exception.AuthenticationException as e:
+ raise e
+ finally:
+ ssh.close()
+
+ return True
+
+ def _detect_hardware_and_os(self, ssh):
+ """Detect the target hardware capabilities and OS series.
+
+ :param object ssh: The SSHClient
+ :return: str: A raw string containing OS and hardware information.
+ """
+
+ info = {
+ 'series': '',
+ 'arch': '',
+ 'cpu-cores': '',
+ 'mem': '',
+ }
+
+ stdout, stderr = self._run_command(
+ ssh,
+ ["sudo", "/bin/bash -c " + shlex.quote(DETECTION_SCRIPT)],
+ pty=True,
+ )
+
+ lines = stdout.split("\n")
+ info['series'] = lines[0].strip()
+ info['arch'] = normalize_arch(lines[1].strip())
+
+ memKb = re.split(r'\s+', lines[2])[1]
+
+ # Convert megabytes -> kilobytes
+ info['mem'] = round(int(memKb) / 1024)
+
+ # Detect available CPUs
+ recorded = {}
+ for line in lines[3:]:
+ physical_id = ""
+ print(line)
+
+ if line.find("physical id") == 0:
+ physical_id = line.split(":")[1].strip()
+ elif line.find("cpu cores") == 0:
+ cores = line.split(":")[1].strip()
+
+ if physical_id not in recorded.keys():
+ info['cpu-cores'] += cores
+ recorded[physical_id] = True
+
+ return info
+
+ def provision_machine(self):
+ """Perform the initial provisioning of the target machine.
+
+ :return: bool: The client.AddMachineParams
+ :raises: :class:`paramiko.ssh_exception.AuthenticationException`
+ if the upload fails
+ """
+ params = client.AddMachineParams()
+
+ if self._init_ubuntu_user():
+ try:
+
+ ssh = self._get_ssh_client(
+ self.host,
+ self.user,
+ self.private_key_path
+ )
+
+ hw = self._detect_hardware_and_os(ssh)
+ params.series = hw['series']
+ params.instance_id = "manual:{}".format(self.host)
+ params.nonce = "manual:{}:{}".format(
+ self.host,
+ str(uuid.uuid4()), # a nop for Juju w/manual machines
+ )
+ params.hardware_characteristics = {
+ 'arch': hw['arch'],
+ 'mem': int(hw['mem']),
+ 'cpu-cores': int(hw['cpu-cores']),
+ }
+ params.addresses = [{
+ 'value': self.host,
+ 'type': 'ipv4',
+ 'scope': 'public',
+ }]
+
+ except paramiko.ssh_exception.AuthenticationException as e:
+ raise e
+ finally:
+ ssh.close()
+
+ return params
+
+ async def install_agent(self, connection, nonce, machine_id):
+ """
+ :param object connection: Connection to Juju API
+ :param str nonce: The nonce machine specification
+ :param str machine_id: The id assigned to the machine
+
+ :return: bool: If the initialization was successful
+ """
+
+ # The path where the Juju agent should be installed.
+ data_dir = "/var/lib/juju"
+
+ # Disabling this prevents `apt-get update` from running initially, so
+ # charms will fail to deploy
+ disable_package_commands = False
+
+ client_facade = client.ClientFacade.from_connection(connection)
+ results = await client_facade.ProvisioningScript(
+ data_dir,
+ disable_package_commands,
+ machine_id,
+ nonce,
+ )
+
+ self._run_configure_script(results.script)
+
+ def _run_configure_script(self, script):
+ """Run the script to install the Juju agent on the target machine.
+
+ :param str script: The script returned by the ProvisioningScript API
+ :raises: :class:`paramiko.ssh_exception.AuthenticationException`
+ if the upload fails
+ """
+
+ _, tmpFile = tempfile.mkstemp()
+ with open(tmpFile, 'w') as f:
+ f.write(script)
+
+ try:
+ # get ssh client
+ ssh = self._get_ssh_client(
+ self.host,
+ "ubuntu",
+ self.private_key_path,
+ )
+
+ # copy the local copy of the script to the remote machine
+ sftp = paramiko.SFTPClient.from_transport(ssh.get_transport())
+ sftp.put(
+ tmpFile,
+ tmpFile,
+ )
+
+ # run the provisioning script
+ stdout, stderr = self._run_command(
+ ssh,
+ "sudo /bin/bash {}".format(tmpFile),
+ )
+
+ except paramiko.ssh_exception.AuthenticationException as e:
+ raise e
+ finally:
+ os.remove(tmpFile)
+ ssh.close()