modules/libjuju/juju/provisioner.py

   1 from .client import client
   2
   3 import paramiko
   4 import os
   5 import re
   6 import tempfile
   7 import shlex
   8 from subprocess import (
   9     CalledProcessError,
  10 )
  11 import uuid
  12
  13
  14 arches = [
  15     [re.compile(r"amd64|x86_64"), "amd64"],
  16     [re.compile(r"i?[3-9]86"), "i386"],
  17     [re.compile(r"(arm$)|(armv.*)"), "armhf"],
  18     [re.compile(r"aarch64"), "arm64"],
  19     [re.compile(r"ppc64|ppc64el|ppc64le"), "ppc64el"],
  20     [re.compile(r"s390x?"), "s390x"],
  21 ]
  22
  23
  24 def normalize_arch(rawArch):
  25     """Normalize the architecture string."""
  26     for arch in arches:
  27         if arch[0].match(rawArch):
  28             return arch[1]
  29
  30
  31 DETECTION_SCRIPT = """#!/bin/bash
  32 set -e
  33 os_id=$(grep '^ID=' /etc/os-release | tr -d '"' | cut -d= -f2)
  34 if [ "$os_id" = 'centos' ]; then
  35   os_version=$(grep '^VERSION_ID=' /etc/os-release | tr -d '"' | cut -d= -f2)
  36   echo "centos$os_version"
  37 else
  38   lsb_release -cs
  39 fi
  40 uname -m
  41 grep MemTotal /proc/meminfo
  42 cat /proc/cpuinfo
  43 """
  44
  45 INITIALIZE_UBUNTU_SCRIPT = """set -e
  46 (id ubuntu &> /dev/null) || useradd -m ubuntu -s /bin/bash
  47 umask 0077
  48 temp=$(mktemp)
  49 echo 'ubuntu ALL=(ALL) NOPASSWD:ALL' > $temp
  50 install -m 0440 $temp /etc/sudoers.d/90-juju-ubuntu
  51 rm $temp
  52 su ubuntu -c 'install -D -m 0600 /dev/null ~/.ssh/authorized_keys'
  53 export authorized_keys="{}"
  54 if [ ! -z "$authorized_keys" ]; then
  55     su ubuntu -c 'echo $authorized_keys >> ~/.ssh/authorized_keys'
  56 fi
  57 """
  58
  59
  60 class SSHProvisioner:
  61     """Provision a manually created machine via SSH."""
  62     user = ""
  63     host = ""
  64     private_key_path = ""
  65
  66     def __init__(self, user, host, private_key_path):
  67         self.host = host
  68         self.user = user
  69         self.private_key_path = private_key_path
  70
  71     def _get_ssh_client(self, host, user, key):
  72         """Return a connected Paramiko ssh object.
  73
  74         :param str host: The host to connect to.
  75         :param str user: The user to connect as.
  76         :param str key: The private key to authenticate with.
  77
  78         :return: object: A paramiko.SSHClient
  79         :raises: :class:`paramiko.ssh_exception.SSHException` if the
  80             connection failed
  81         """
  82
  83         ssh = paramiko.SSHClient()
  84         ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
  85
  86         pkey = None
  87
  88         # Read the private key into a paramiko.RSAKey
  89         if os.path.exists(key):
  90             with open(key, 'r') as f:
  91                 pkey = paramiko.RSAKey.from_private_key(f)
  92
  93         #######################################################################
  94         # There is a bug in some versions of OpenSSH 4.3 (CentOS/RHEL5) where #
  95         # the server may not send the SSH_MSG_USERAUTH_BANNER message except  #
  96         # when responding to an auth_none request. For example, paramiko will #
  97         # attempt to use password authentication when a password is set, but  #
  98         # the server could deny that, instead requesting keyboard-interactive.#
  99         # The hack to workaround this is to attempt a reconnect, which will   #
 100         # receive the right banner, and authentication can proceed. See the   #
 101         # following for more info:                                            #
 102         # https://github.com/paramiko/paramiko/issues/432                     #
 103         # https://github.com/paramiko/paramiko/pull/438                       #
 104         #######################################################################
 105
 106         try:
 107             ssh.connect(host, port=22, username=user, pkey=pkey)
 108         except paramiko.ssh_exception.SSHException as e:
 109             if 'Error reading SSH protocol banner' == str(e):
 110                 # Once more, with feeling
 111                 ssh.connect(host, port=22, username=user, pkey=pkey)
 112             else:
 113                 # Reraise the original exception
 114                 raise e
 115
 116         return ssh
 117
 118     def _run_command(self, ssh, cmd, pty=True):
 119         """Run a command remotely via SSH.
 120
 121         :param object ssh: The SSHClient
 122         :param str cmd: The command to execute
 123         :param list cmd: The `shlex.split` command to execute
 124         :param bool pty: Whether to allocate a pty
 125
 126         :return: tuple: The stdout and stderr of the command execution
 127         :raises: :class:`CalledProcessError` if the command fails
 128         """
 129
 130         if isinstance(cmd, str):
 131             cmd = shlex.split(cmd)
 132
 133         if type(cmd) is not list:
 134             cmd = [cmd]
 135
 136         cmds = ' '.join(cmd)
 137         stdin, stdout, stderr = ssh.exec_command(cmds, get_pty=pty)
 138         retcode = stdout.channel.recv_exit_status()
 139
 140         if retcode > 0:
 141             output = stderr.read().strip()
 142             raise CalledProcessError(returncode=retcode, cmd=cmd,
 143                                      output=output)
 144         return (
 145             stdout.read().decode('utf-8').strip(),
 146             stderr.read().decode('utf-8').strip()
 147         )
 148
 149     def _init_ubuntu_user(self):
 150         """Initialize the ubuntu user.
 151
 152         :return: bool: If the initialization was successful
 153         :raises: :class:`paramiko.ssh_exception.AuthenticationException`
 154             if the authentication fails
 155         """
 156
 157         # TODO: Test this on an image without the ubuntu user setup.
 158
 159         auth_user = self.user
 160         ssh = None
 161         try:
 162             # Run w/o allocating a pty, so we fail if sudo prompts for a passwd
 163             ssh = self._get_ssh_client(
 164                 self.host,
 165                 "ubuntu",
 166                 self.private_key_path,
 167             )
 168
 169             stdout, stderr = self._run_command(ssh, "sudo -n true", pty=False)
 170         except paramiko.ssh_exception.AuthenticationException as e:
 171             raise e
 172         else:
 173             auth_user = "ubuntu"
 174         finally:
 175             if ssh:
 176                 ssh.close()
 177
 178         # if the above fails, run the init script as the authenticated user
 179
 180         # Infer the public key
 181         public_key = None
 182         public_key_path = "{}.pub".format(self.private_key_path)
 183
 184         if not os.path.exists(public_key_path):
 185             raise FileNotFoundError(
 186                 "Public key '{}' doesn't exist.".format(public_key_path)
 187             )
 188
 189         with open(public_key_path, "r") as f:
 190             public_key = f.readline()
 191
 192         script = INITIALIZE_UBUNTU_SCRIPT.format(public_key)
 193
 194         try:
 195             ssh = self._get_ssh_client(
 196                 self.host,
 197                 auth_user,
 198                 self.private_key_path,
 199             )
 200
 201             self._run_command(
 202                 ssh,
 203                 ["sudo", "/bin/bash -c " + shlex.quote(script)],
 204                 pty=True
 205             )
 206         except paramiko.ssh_exception.AuthenticationException as e:
 207             raise e
 208         finally:
 209             ssh.close()
 210
 211         return True
 212
 213     def _detect_hardware_and_os(self, ssh):
 214         """Detect the target hardware capabilities and OS series.
 215
 216         :param object ssh: The SSHClient
 217         :return: str: A raw string containing OS and hardware information.
 218         """
 219
 220         info = {
 221             'series': '',
 222             'arch': '',
 223             'cpu-cores': '',
 224             'mem': '',
 225         }
 226
 227         stdout, stderr = self._run_command(
 228             ssh,
 229             ["sudo", "/bin/bash -c " + shlex.quote(DETECTION_SCRIPT)],
 230             pty=True,
 231         )
 232
 233         lines = stdout.split("\n")
 234         info['series'] = lines[0].strip()
 235         info['arch'] = normalize_arch(lines[1].strip())
 236
 237         memKb = re.split(r'\s+', lines[2])[1]
 238
 239         # Convert megabytes -> kilobytes
 240         info['mem'] = round(int(memKb) / 1024)
 241
 242         # Detect available CPUs
 243         recorded = {}
 244         for line in lines[3:]:
 245             physical_id = ""
 246             print(line)
 247
 248             if line.find("physical id") == 0:
 249                 physical_id = line.split(":")[1].strip()
 250             elif line.find("cpu cores") == 0:
 251                 cores = line.split(":")[1].strip()
 252
 253                 if physical_id not in recorded.keys():
 254                     info['cpu-cores'] += cores
 255                     recorded[physical_id] = True
 256
 257         return info
 258
 259     def provision_machine(self):
 260         """Perform the initial provisioning of the target machine.
 261
 262         :return: bool: The client.AddMachineParams
 263         :raises: :class:`paramiko.ssh_exception.AuthenticationException`
 264             if the upload fails
 265         """
 266         params = client.AddMachineParams()
 267
 268         if self._init_ubuntu_user():
 269             try:
 270
 271                 ssh = self._get_ssh_client(
 272                     self.host,
 273                     self.user,
 274                     self.private_key_path
 275                 )
 276
 277                 hw = self._detect_hardware_and_os(ssh)
 278                 params.series = hw['series']
 279                 params.instance_id = "manual:{}".format(self.host)
 280                 params.nonce = "manual:{}:{}".format(
 281                     self.host,
 282                     str(uuid.uuid4()),  # a nop for Juju w/manual machines
 283                 )
 284                 params.hardware_characteristics = {
 285                     'arch': hw['arch'],
 286                     'mem': int(hw['mem']),
 287                     'cpu-cores': int(hw['cpu-cores']),
 288                 }
 289                 params.addresses = [{
 290                     'value': self.host,
 291                     'type': 'ipv4',
 292                     'scope': 'public',
 293                 }]
 294
 295             except paramiko.ssh_exception.AuthenticationException as e:
 296                 raise e
 297             finally:
 298                 ssh.close()
 299
 300         return params
 301
 302     async def install_agent(self, connection, nonce, machine_id):
 303         """
 304         :param object connection: Connection to Juju API
 305         :param str nonce: The nonce machine specification
 306         :param str machine_id: The id assigned to the machine
 307
 308         :return: bool: If the initialization was successful
 309         """
 310
 311         # The path where the Juju agent should be installed.
 312         data_dir = "/var/lib/juju"
 313
 314         # Disabling this prevents `apt-get update` from running initially, so
 315         # charms will fail to deploy
 316         disable_package_commands = False
 317
 318         client_facade = client.ClientFacade.from_connection(connection)
 319         results = await client_facade.ProvisioningScript(
 320             data_dir,
 321             disable_package_commands,
 322             machine_id,
 323             nonce,
 324         )
 325
 326         self._run_configure_script(results.script)
 327
 328     def _run_configure_script(self, script):
 329         """Run the script to install the Juju agent on the target machine.
 330
 331         :param str script: The script returned by the ProvisioningScript API
 332         :raises: :class:`paramiko.ssh_exception.AuthenticationException`
 333             if the upload fails
 334         """
 335
 336         _, tmpFile = tempfile.mkstemp()
 337         with open(tmpFile, 'w') as f:
 338             f.write(script)
 339
 340         try:
 341             # get ssh client
 342             ssh = self._get_ssh_client(
 343                 self.host,
 344                 "ubuntu",
 345                 self.private_key_path,
 346             )
 347
 348             # copy the local copy of the script to the remote machine
 349             sftp = paramiko.SFTPClient.from_transport(ssh.get_transport())
 350             sftp.put(
 351                 tmpFile,
 352                 tmpFile,
 353             )
 354
 355             # run the provisioning script
 356             stdout, stderr = self._run_command(
 357                 ssh,
 358                 "sudo /bin/bash {}".format(tmpFile),
 359             )
 360
 361         except paramiko.ssh_exception.AuthenticationException as e:
 362             raise e
 363         finally:
 364             os.remove(tmpFile)
 365             ssh.close()