modules/libjuju/juju/provisioner.py

   1 # Copyright 2019 Canonical Ltd.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 #     Unless required by applicable law or agreed to in writing, software
  10 #     distributed under the License is distributed on an "AS IS" BASIS,
  11 #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 #     See the License for the specific language governing permissions and
  13 #     limitations under the License.
  14 import os
  15 import re
  16 import shlex
  17 import tempfile
  18 import uuid
  19 from subprocess import CalledProcessError
  20
  21 import paramiko
  22
  23 from .client import client
  24
  25 arches = [
  26     [re.compile(r"amd64|x86_64"), "amd64"],
  27     [re.compile(r"i?[3-9]86"), "i386"],
  28     [re.compile(r"(arm$)|(armv.*)"), "armhf"],
  29     [re.compile(r"aarch64"), "arm64"],
  30     [re.compile(r"ppc64|ppc64el|ppc64le"), "ppc64el"],
  31     [re.compile(r"s390x?"), "s390x"],
  32
  33 ]
  34
  35
  36 def normalize_arch(rawArch):
  37     """Normalize the architecture string."""
  38     for arch in arches:
  39         if arch[0].match(rawArch):
  40             return arch[1]
  41
  42
  43 DETECTION_SCRIPT = """#!/bin/bash
  44 set -e
  45 os_id=$(grep '^ID=' /etc/os-release | tr -d '"' | cut -d= -f2)
  46 if [ "$os_id" = 'centos' ]; then
  47   os_version=$(grep '^VERSION_ID=' /etc/os-release | tr -d '"' | cut -d= -f2)
  48   echo "centos$os_version"
  49 else
  50   lsb_release -cs
  51 fi
  52 uname -m
  53 grep MemTotal /proc/meminfo
  54 cat /proc/cpuinfo
  55 """
  56
  57 INITIALIZE_UBUNTU_SCRIPT = """set -e
  58 (id ubuntu &> /dev/null) || useradd -m ubuntu -s /bin/bash
  59 umask 0077
  60 temp=$(mktemp)
  61 echo 'ubuntu ALL=(ALL) NOPASSWD:ALL' > $temp
  62 install -m 0440 $temp /etc/sudoers.d/90-juju-ubuntu
  63 rm $temp
  64 su ubuntu -c 'install -D -m 0600 /dev/null ~/.ssh/authorized_keys'
  65 export authorized_keys="{}"
  66 if [ ! -z "$authorized_keys" ]; then
  67     su ubuntu -c 'echo $authorized_keys >> ~/.ssh/authorized_keys'
  68 fi
  69 """
  70
  71
  72 class SSHProvisioner:
  73     """Provision a manually created machine via SSH."""
  74     user = ""
  75     host = ""
  76     private_key_path = ""
  77
  78     def __init__(self, user, host, private_key_path):
  79         self.host = host
  80         self.user = user
  81         self.private_key_path = private_key_path
  82
  83     def _get_ssh_client(self, host, user, key):
  84         """Return a connected Paramiko ssh object.
  85
  86         :param str host: The host to connect to.
  87         :param str user: The user to connect as.
  88         :param str key: The private key to authenticate with.
  89
  90         :return: object: A paramiko.SSHClient
  91         :raises: :class:`paramiko.ssh_exception.SSHException` if the
  92             connection failed
  93         """
  94
  95         ssh = paramiko.SSHClient()
  96         ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
  97
  98         pkey = None
  99
 100         # Read the private key into a paramiko.RSAKey
 101         if os.path.exists(key):
 102             with open(key, 'r') as f:
 103                 pkey = paramiko.RSAKey.from_private_key(f)
 104
 105         #######################################################################
 106         # There is a bug in some versions of OpenSSH 4.3 (CentOS/RHEL5) where #
 107         # the server may not send the SSH_MSG_USERAUTH_BANNER message except  #
 108         # when responding to an auth_none request. For example, paramiko will #
 109         # attempt to use password authentication when a password is set, but  #
 110         # the server could deny that, instead requesting keyboard-interactive.#
 111         # The hack to workaround this is to attempt a reconnect, which will   #
 112         # receive the right banner, and authentication can proceed. See the   #
 113         # following for more info:                                            #
 114         # https://github.com/paramiko/paramiko/issues/432                     #
 115         # https://github.com/paramiko/paramiko/pull/438                       #
 116         #######################################################################
 117
 118         try:
 119             ssh.connect(host, port=22, username=user, pkey=pkey)
 120         except paramiko.ssh_exception.SSHException as e:
 121             if 'Error reading SSH protocol banner' == str(e):
 122                 # Once more, with feeling
 123                 ssh.connect(host, port=22, username=user, pkey=pkey)
 124             else:
 125                 # Reraise the original exception
 126                 raise e
 127
 128         return ssh
 129
 130     def _run_command(self, ssh, cmd, pty=True):
 131         """Run a command remotely via SSH.
 132
 133         :param object ssh: The SSHClient
 134         :param str cmd: The command to execute
 135         :param list cmd: The `shlex.split` command to execute
 136         :param bool pty: Whether to allocate a pty
 137
 138         :return: tuple: The stdout and stderr of the command execution
 139         :raises: :class:`CalledProcessError` if the command fails
 140         """
 141
 142         if isinstance(cmd, str):
 143             cmd = shlex.split(cmd)
 144
 145         if type(cmd) is not list:
 146             cmd = [cmd]
 147
 148         cmds = ' '.join(cmd)
 149         stdin, stdout, stderr = ssh.exec_command(cmds, get_pty=pty)
 150         retcode = stdout.channel.recv_exit_status()
 151
 152         if retcode > 0:
 153             output = stderr.read().strip()
 154             raise CalledProcessError(returncode=retcode, cmd=cmd,
 155                                      output=output)
 156         return (
 157             stdout.read().decode('utf-8').strip(),
 158             stderr.read().decode('utf-8').strip()
 159         )
 160
 161     def _init_ubuntu_user(self):
 162         """Initialize the ubuntu user.
 163
 164         :return: bool: If the initialization was successful
 165         :raises: :class:`paramiko.ssh_exception.AuthenticationException`
 166             if the authentication fails
 167         """
 168
 169         ssh = None
 170         try:
 171             # Run w/o allocating a pty, so we fail if sudo prompts for a passwd
 172             ssh = self._get_ssh_client(
 173                 self.host,
 174                 self.user,
 175                 self.private_key_path,
 176             )
 177             stdout, stderr = self._run_command(ssh, "sudo -n true", pty=False)
 178         except paramiko.ssh_exception.AuthenticationException as e:
 179             raise e
 180         finally:
 181             if ssh:
 182                 ssh.close()
 183
 184         # Infer the public key
 185         public_key = None
 186         public_key_path = "{}.pub".format(self.private_key_path)
 187
 188         if not os.path.exists(public_key_path):
 189             raise FileNotFoundError(
 190                 "Public key '{}' doesn't exist.".format(public_key_path)
 191             )
 192
 193         with open(public_key_path, "r") as f:
 194             public_key = f.readline()
 195
 196         script = INITIALIZE_UBUNTU_SCRIPT.format(public_key)
 197
 198         try:
 199             ssh = self._get_ssh_client(
 200                 self.host,
 201                 self.user,
 202                 self.private_key_path,
 203             )
 204
 205             self._run_command(
 206                 ssh,
 207                 ["sudo", "/bin/bash -c " + shlex.quote(script)],
 208                 pty=True
 209             )
 210         except paramiko.ssh_exception.AuthenticationException as e:
 211             raise e
 212         finally:
 213             ssh.close()
 214
 215         return True
 216
 217     def _detect_hardware_and_os(self, ssh):
 218         """Detect the target hardware capabilities and OS series.
 219
 220         :param object ssh: The SSHClient
 221         :return: str: A raw string containing OS and hardware information.
 222         """
 223
 224         info = {
 225             'series': '',
 226             'arch': '',
 227             'cpu-cores': '',
 228             'mem': '',
 229         }
 230
 231         stdout, stderr = self._run_command(
 232             ssh,
 233             ["sudo", "/bin/bash -c " + shlex.quote(DETECTION_SCRIPT)],
 234             pty=True,
 235         )
 236
 237         lines = stdout.split("\n")
 238         info['series'] = lines[0].strip()
 239         info['arch'] = normalize_arch(lines[1].strip())
 240
 241         memKb = re.split(r'\s+', lines[2])[1]
 242
 243         # Convert megabytes -> kilobytes
 244         info['mem'] = round(int(memKb) / 1024)
 245
 246         # Detect available CPUs
 247         recorded = {}
 248         for line in lines[3:]:
 249             physical_id = ""
 250             print(line)
 251
 252             if line.find("physical id") == 0:
 253                 physical_id = line.split(":")[1].strip()
 254             elif line.find("cpu cores") == 0:
 255                 cores = line.split(":")[1].strip()
 256
 257                 if physical_id not in recorded.keys():
 258                     info['cpu-cores'] += cores
 259                     recorded[physical_id] = True
 260
 261         return info
 262
 263     def provision_machine(self):
 264         """Perform the initial provisioning of the target machine.
 265
 266         :return: bool: The client.AddMachineParams
 267         :raises: :class:`paramiko.ssh_exception.AuthenticationException`
 268             if the upload fails
 269         """
 270         params = client.AddMachineParams()
 271
 272         if self._init_ubuntu_user():
 273             try:
 274
 275                 ssh = self._get_ssh_client(
 276                     self.host,
 277                     self.user,
 278                     self.private_key_path
 279                 )
 280
 281                 hw = self._detect_hardware_and_os(ssh)
 282                 params.series = hw['series']
 283                 params.instance_id = "manual:{}".format(self.host)
 284                 params.nonce = "manual:{}:{}".format(
 285                     self.host,
 286                     str(uuid.uuid4()),  # a nop for Juju w/manual machines
 287                 )
 288                 params.hardware_characteristics = {
 289                     'arch': hw['arch'],
 290                     'mem': int(hw['mem']),
 291                     'cpu-cores': int(hw['cpu-cores']),
 292                 }
 293                 params.addresses = [{
 294                     'value': self.host,
 295                     'type': 'ipv4',
 296                     'scope': 'public',
 297                 }]
 298
 299             except paramiko.ssh_exception.AuthenticationException as e:
 300                 raise e
 301             finally:
 302                 ssh.close()
 303
 304         return params
 305
 306     async def install_agent(self, connection, nonce, machine_id):
 307         """
 308         :param object connection: Connection to Juju API
 309         :param str nonce: The nonce machine specification
 310         :param str machine_id: The id assigned to the machine
 311
 312         :return: bool: If the initialization was successful
 313         """
 314
 315         # The path where the Juju agent should be installed.
 316         data_dir = "/var/lib/juju"
 317
 318         # Disabling this prevents `apt-get update` from running initially, so
 319         # charms will fail to deploy
 320         disable_package_commands = False
 321
 322         client_facade = client.ClientFacade.from_connection(connection)
 323         results = await client_facade.ProvisioningScript(
 324             data_dir=data_dir,
 325             disable_package_commands=disable_package_commands,
 326             machine_id=machine_id,
 327             nonce=nonce,
 328         )
 329
 330         self._run_configure_script(results.script)
 331
 332     def _run_configure_script(self, script):
 333         """Run the script to install the Juju agent on the target machine.
 334
 335         :param str script: The script returned by the ProvisioningScript API
 336         :raises: :class:`paramiko.ssh_exception.AuthenticationException`
 337             if the upload fails
 338         """
 339
 340         _, tmpFile = tempfile.mkstemp()
 341         with open(tmpFile, 'w') as f:
 342             f.write(script)
 343
 344         try:
 345             # get ssh client
 346             ssh = self._get_ssh_client(
 347                 self.host,
 348                 "ubuntu",
 349                 self.private_key_path,
 350             )
 351
 352             # copy the local copy of the script to the remote machine
 353             sftp = paramiko.SFTPClient.from_transport(ssh.get_transport())
 354             sftp.put(
 355                 tmpFile,
 356                 tmpFile,
 357             )
 358
 359             # run the provisioning script
 360             stdout, stderr = self._run_command(
 361                 ssh,
 362                 "sudo /bin/bash {}".format(tmpFile),
 363             )
 364
 365         except paramiko.ssh_exception.AuthenticationException as e:
 366             raise e
 367         finally:
 368             os.remove(tmpFile)
 369             ssh.close()