modules/libjuju/juju/provisioner.py

   1 from .client import client
   2
   3 import paramiko
   4 import os
   5 import re
   6 import tempfile
   7 import shlex
   8 from subprocess import (
   9     CalledProcessError,
  10 )
  11 import uuid
  12
  13
  14 arches = [
  15     [re.compile("amd64|x86_64"), "amd64"],
  16     [re.compile("i?[3-9]86"), "i386"],
  17     [re.compile("(arm$)|(armv.*)"), "armhf"],
  18     [re.compile("aarch64"), "arm64"],
  19     [re.compile("ppc64|ppc64el|ppc64le"), "ppc64el"],
  20     [re.compile("ppc64|ppc64el|ppc64le"), "s390x"],
  21
  22 ]
  23
  24
  25 def normalize_arch(rawArch):
  26     """Normalize the architecture string."""
  27     for arch in arches:
  28         if arch[0].match(rawArch):
  29             return arch[1]
  30
  31
  32 DETECTION_SCRIPT = """#!/bin/bash
  33 set -e
  34 os_id=$(grep '^ID=' /etc/os-release | tr -d '"' | cut -d= -f2)
  35 if [ "$os_id" = 'centos' ]; then
  36   os_version=$(grep '^VERSION_ID=' /etc/os-release | tr -d '"' | cut -d= -f2)
  37   echo "centos$os_version"
  38 else
  39   lsb_release -cs
  40 fi
  41 uname -m
  42 grep MemTotal /proc/meminfo
  43 cat /proc/cpuinfo
  44 """
  45
  46 INITIALIZE_UBUNTU_SCRIPT = """set -e
  47 (id ubuntu &> /dev/null) || useradd -m ubuntu -s /bin/bash
  48 umask 0077
  49 temp=$(mktemp)
  50 echo 'ubuntu ALL=(ALL) NOPASSWD:ALL' > $temp
  51 install -m 0440 $temp /etc/sudoers.d/90-juju-ubuntu
  52 rm $temp
  53 su ubuntu -c 'install -D -m 0600 /dev/null ~/.ssh/authorized_keys'
  54 export authorized_keys="{}"
  55 if [ ! -z "$authorized_keys" ]; then
  56     su ubuntu -c 'echo $authorized_keys >> ~/.ssh/authorized_keys'
  57 fi
  58 """
  59
  60
  61 class SSHProvisioner:
  62     """Provision a manually created machine via SSH."""
  63     user = ""
  64     host = ""
  65     private_key_path = ""
  66
  67     def __init__(self, user, host, private_key_path):
  68         self.host = host
  69         self.user = user
  70         self.private_key_path = private_key_path
  71
  72     def _get_ssh_client(self, host, user, key):
  73         """Return a connected Paramiko ssh object.
  74
  75         :param str host: The host to connect to.
  76         :param str user: The user to connect as.
  77         :param str key: The private key to authenticate with.
  78
  79         :return: object: A paramiko.SSHClient
  80         :raises: :class:`paramiko.ssh_exception.SSHException` if the
  81             connection failed
  82         """
  83
  84         ssh = paramiko.SSHClient()
  85         ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
  86
  87         pkey = None
  88
  89         # Read the private key into a paramiko.RSAKey
  90         if os.path.exists(key):
  91             with open(key, 'r') as f:
  92                 pkey = paramiko.RSAKey.from_private_key(f)
  93
  94         #######################################################################
  95         # There is a bug in some versions of OpenSSH 4.3 (CentOS/RHEL5) where #
  96         # the server may not send the SSH_MSG_USERAUTH_BANNER message except  #
  97         # when responding to an auth_none request. For example, paramiko will #
  98         # attempt to use password authentication when a password is set, but  #
  99         # the server could deny that, instead requesting keyboard-interactive.#
 100         # The hack to workaround this is to attempt a reconnect, which will   #
 101         # receive the right banner, and authentication can proceed. See the   #
 102         # following for more info:                                            #
 103         # https://github.com/paramiko/paramiko/issues/432                     #
 104         # https://github.com/paramiko/paramiko/pull/438                       #
 105         #######################################################################
 106
 107         try:
 108             ssh.connect(host, port=22, username=user, pkey=pkey)
 109         except paramiko.ssh_exception.SSHException as e:
 110             if 'Error reading SSH protocol banner' == str(e):
 111                 # Once more, with feeling
 112                 ssh.connect(host, port=22, username=user, pkey=pkey)
 113             else:
 114                 # Reraise the original exception
 115                 raise e
 116
 117         return ssh
 118
 119     def _run_command(self, ssh, cmd, pty=True):
 120         """Run a command remotely via SSH.
 121
 122         :param object ssh: The SSHClient
 123         :param str cmd: The command to execute
 124         :param list cmd: The `shlex.split` command to execute
 125         :param bool pty: Whether to allocate a pty
 126
 127         :return: tuple: The stdout and stderr of the command execution
 128         :raises: :class:`CalledProcessError` if the command fails
 129         """
 130
 131         if isinstance(cmd, str):
 132             cmd = shlex.split(cmd)
 133
 134         if type(cmd) is not list:
 135             cmd = [cmd]
 136
 137         cmds = ' '.join(cmd)
 138         stdin, stdout, stderr = ssh.exec_command(cmds, get_pty=pty)
 139         retcode = stdout.channel.recv_exit_status()
 140
 141         if retcode > 0:
 142             output = stderr.read().strip()
 143             raise CalledProcessError(returncode=retcode, cmd=cmd,
 144                                      output=output)
 145         return (
 146             stdout.read().decode('utf-8').strip(),
 147             stderr.read().decode('utf-8').strip()
 148         )
 149
 150     def _init_ubuntu_user(self):
 151         """Initialize the ubuntu user.
 152
 153         :return: bool: If the initialization was successful
 154         :raises: :class:`paramiko.ssh_exception.AuthenticationException`
 155             if the authentication fails
 156         """
 157
 158         # TODO: Test this on an image without the ubuntu user setup.
 159
 160         auth_user = self.user
 161         try:
 162             # Run w/o allocating a pty, so we fail if sudo prompts for a passwd
 163             ssh = self._get_ssh_client(
 164                 self.host,
 165                 "ubuntu",
 166                 self.private_key_path,
 167             )
 168
 169             stdout, stderr = self._run_command(ssh, "sudo -n true", pty=False)
 170         except paramiko.ssh_exception.AuthenticationException as e:
 171             raise e
 172         else:
 173             auth_user = "ubuntu"
 174         finally:
 175             if ssh:
 176                 ssh.close()
 177
 178         # if the above fails, run the init script as the authenticated user
 179
 180         # Infer the public key
 181         public_key = None
 182         public_key_path = "{}.pub".format(self.private_key_path)
 183
 184         if not os.path.exists(public_key_path):
 185             raise FileNotFoundError(
 186                 "Public key '{}' doesn't exist.".format(public_key_path)
 187             )
 188
 189         with open(public_key_path, "r") as f:
 190             public_key = f.readline()
 191
 192         script = INITIALIZE_UBUNTU_SCRIPT.format(public_key)
 193
 194         try:
 195             ssh = self._get_ssh_client(
 196                 self.host,
 197                 auth_user,
 198                 self.private_key_path,
 199             )
 200
 201             self._run_command(
 202                 ssh,
 203                 ["sudo", "/bin/bash -c " + shlex.quote(script)],
 204                 pty=True
 205             )
 206         except paramiko.ssh_exception.AuthenticationException as e:
 207             raise e
 208         finally:
 209             ssh.close()
 210
 211         return True
 212
 213     def _detect_hardware_and_os(self, ssh):
 214         """Detect the target hardware capabilities and OS series.
 215
 216         :param object ssh: The SSHClient
 217         :return: str: A raw string containing OS and hardware information.
 218         """
 219
 220         info = {
 221             'series': '',
 222             'arch': '',
 223             'cpu-cores': '',
 224             'mem': '',
 225         }
 226
 227         stdout, stderr = self._run_command(
 228             ssh,
 229             ["sudo", "/bin/bash -c " + shlex.quote(DETECTION_SCRIPT)],
 230             pty=True,
 231         )
 232
 233         lines = stdout.split("\n")
 234         info['series'] = lines[0].strip()
 235         info['arch'] = normalize_arch(lines[1].strip())
 236
 237         memKb = re.split('\s+', lines[2])[1]
 238
 239         # Convert megabytes -> kilobytes
 240         info['mem'] = round(int(memKb) / 1024)
 241
 242         # Detect available CPUs
 243         recorded = {}
 244         for line in lines[3:]:
 245             physical_id = ""
 246             print(line)
 247
 248             if line.find("physical id") == 0:
 249                 physical_id = line.split(":")[1].strip()
 250             elif line.find("cpu cores") == 0:
 251                 cores = line.split(":")[1].strip()
 252
 253                 if physical_id not in recorded.keys():
 254                         info['cpu-cores'] += cores
 255                         recorded[physical_id] = True
 256
 257         return info
 258
 259     def provision_machine(self):
 260         """Perform the initial provisioning of the target machine.
 261
 262         :return: bool: The client.AddMachineParams
 263         :raises: :class:`paramiko.ssh_exception.AuthenticationException`
 264             if the upload fails
 265         """
 266         params = client.AddMachineParams()
 267
 268         if self._init_ubuntu_user():
 269             try:
 270
 271                 ssh = self._get_ssh_client(
 272                     self.host,
 273                     self.user,
 274                     self.private_key_path
 275                 )
 276
 277                 hw = self._detect_hardware_and_os(ssh)
 278                 params.series = hw['series']
 279                 params.instance_id = "manual:{}".format(self.host)
 280                 params.nonce = "manual:{}:{}".format(
 281                     self.host,
 282                     str(uuid.uuid4()),  # a nop for Juju w/manual machines
 283                 )
 284                 params.hardware_characteristics = {
 285                     'arch': hw['arch'],
 286                     'mem': int(hw['mem']),
 287                     'cpu-cores': int(hw['cpu-cores']),
 288                 }
 289                 params.addresses = [{
 290                     'value': self.host,
 291                     'type': 'ipv4',
 292                     'scope': 'public',
 293                 }]
 294
 295             except paramiko.ssh_exception.AuthenticationException as e:
 296                 raise e
 297             finally:
 298                 ssh.close()
 299
 300         return params
 301
 302     async def install_agent(self, connection, nonce, machine_id):
 303         """
 304         :param object connection: Connection to Juju API
 305         :param str nonce: The nonce machine specification
 306         :param str machine_id: The id assigned to the machine
 307
 308         :return: bool: If the initialization was successful
 309         """
 310
 311         # The path where the Juju agent should be installed.
 312         data_dir = "/var/lib/juju"
 313
 314         # Disabling this prevents `apt-get update` from running initially, so
 315         # charms will fail to deploy
 316         disable_package_commands = False
 317
 318         client_facade = client.ClientFacade.from_connection(connection)
 319         results = await client_facade.ProvisioningScript(
 320             data_dir,
 321             disable_package_commands,
 322             machine_id,
 323             nonce,
 324         )
 325
 326         self._run_configure_script(results.script)
 327
 328     def _run_configure_script(self, script):
 329         """Run the script to install the Juju agent on the target machine.
 330
 331         :param str script: The script returned by the ProvisioningScript API
 332         :raises: :class:`paramiko.ssh_exception.AuthenticationException`
 333             if the upload fails
 334         """
 335
 336         _, tmpFile = tempfile.mkstemp()
 337         with open(tmpFile, 'w') as f:
 338             f.write(script)
 339
 340         try:
 341             # get ssh client
 342             ssh = self._get_ssh_client(
 343                 self.host,
 344                 "ubuntu",
 345                 self.private_key_path,
 346             )
 347
 348             # copy the local copy of the script to the remote machine
 349             sftp = paramiko.SFTPClient.from_transport(ssh.get_transport())
 350             sftp.put(
 351                 tmpFile,
 352                 tmpFile,
 353             )
 354
 355             # run the provisioning script
 356             stdout, stderr = self._run_command(
 357                 ssh,
 358                 "sudo /bin/bash {}".format(tmpFile),
 359             )
 360
 361         except paramiko.ssh_exception.AuthenticationException as e:
 362             raise e
 363         finally:
 364             os.remove(tmpFile)
 365             ssh.close()