n2vc/provisioner.py

   1 # Copyright 2019 Canonical Ltd.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 #     Unless required by applicable law or agreed to in writing, software
  10 #     distributed under the License is distributed on an "AS IS" BASIS,
  11 #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 #     See the License for the specific language governing permissions and
  13 #     limitations under the License.
  14 import logging
  15 import os
  16 import re
  17 from subprocess import CalledProcessError
  18 import tempfile
  19 import uuid
  20
  21 from juju.client import client
  22 import asyncio
  23
  24 arches = [
  25     [re.compile(r"amd64|x86_64"), "amd64"],
  26     [re.compile(r"i?[3-9]86"), "i386"],
  27     [re.compile(r"(arm$)|(armv.*)"), "armhf"],
  28     [re.compile(r"aarch64"), "arm64"],
  29     [re.compile(r"ppc64|ppc64el|ppc64le"), "ppc64el"],
  30     [re.compile(r"s390x?"), "s390x"],
  31 ]
  32
  33
  34 def normalize_arch(rawArch):
  35     """Normalize the architecture string."""
  36     for arch in arches:
  37         if arch[0].match(rawArch):
  38             return arch[1]
  39
  40
  41 DETECTION_SCRIPT = """#!/bin/bash
  42 set -e
  43 os_id=$(grep '^ID=' /etc/os-release | tr -d '"' | cut -d= -f2)
  44 if [ "$os_id" = 'centos' ] || [ "$os_id" = 'rhel' ] ; then
  45   os_version=$(grep '^VERSION_ID=' /etc/os-release | tr -d '"' | cut -d= -f2)
  46   echo "$os_id$os_version"
  47 else
  48   lsb_release -cs
  49 fi
  50 uname -m
  51 grep MemTotal /proc/meminfo
  52 cat /proc/cpuinfo
  53 """
  54
  55 INITIALIZE_UBUNTU_SCRIPT = """set -e
  56 (id ubuntu &> /dev/null) || useradd -m ubuntu -s /bin/bash
  57 umask 0077
  58 temp=$(mktemp)
  59 echo 'ubuntu ALL=(ALL) NOPASSWD:ALL' > $temp
  60 install -m 0440 $temp /etc/sudoers.d/90-juju-ubuntu
  61 rm $temp
  62 su ubuntu -c '[ -f ~/.ssh/authorized_keys ] || install -D -m 0600 /dev/null ~/.ssh/authorized_keys'
  63 export authorized_keys="{}"
  64 if [ ! -z "$authorized_keys" ]; then
  65     su ubuntu -c 'echo $authorized_keys >> ~/.ssh/authorized_keys'
  66 fi
  67 """
  68
  69 IPTABLES_SCRIPT = """#!/bin/bash
  70 set -e
  71 [ -v `which netfilter-persistent` ] && apt update \
  72     && DEBIAN_FRONTEND=noninteractive apt-get install -yqq iptables-persistent
  73 iptables -t nat -A OUTPUT -p tcp -d {} -j DNAT --to-destination {}
  74 netfilter-persistent save
  75 """
  76
  77 IPTABLES_SCRIPT_RHEL = """#!/bin/bash
  78 set -e
  79 [ -v `which firewalld` ] && yum install -q -y firewalld
  80 systemctl is-active --quiet firewalld || systemctl start firewalld \
  81     && firewall-cmd --permanent --zone=public --set-target=ACCEPT
  82 systemctl is-enabled --quiet firewalld || systemctl enable firewalld
  83 firewall-cmd --direct --permanent --add-rule ipv4 nat OUTPUT 0 -d {} -p tcp \
  84     -j DNAT --to-destination {}
  85 firewall-cmd --reload
  86 """
  87
  88 CLOUD_INIT_WAIT_SCRIPT = """#!/bin/bash
  89 set -e
  90 cloud-init status --wait
  91 """
  92
  93
  94 class AsyncSSHProvisioner:
  95     """Provision a manually created machine via SSH."""
  96
  97     user = ""
  98     host = ""
  99     private_key_path = ""
 100
 101     def __init__(self, user, host, private_key_path, log=None):
 102         self.host = host
 103         self.user = user
 104         self.private_key_path = private_key_path
 105         self.log = log if log else logging.getLogger(__name__)
 106
 107     async def _scp(self, source_file, destination_file):
 108         """Execute an scp command. Requires a fully qualified source and
 109         destination.
 110
 111         :param str source_file: Path to the source file
 112         :param str destination_file: Path to the destination file
 113         """
 114         cmd = [
 115             "scp",
 116             "-i",
 117             os.path.expanduser(self.private_key_path),
 118             "-o",
 119             "StrictHostKeyChecking=no",
 120             "-q",
 121             "-B",
 122         ]
 123         destination = "{}@{}:{}".format(self.user, self.host, destination_file)
 124         cmd.extend([source_file, destination])
 125         process = await asyncio.create_subprocess_exec(*cmd)
 126         await process.wait()
 127         if process.returncode != 0:
 128             raise CalledProcessError(returncode=process.returncode, cmd=cmd)
 129
 130     async def _ssh(self, command):
 131         """Run a command remotely via SSH.
 132
 133         :param str command: The command to execute
 134         :return: tuple: The stdout and stderr of the command execution
 135         :raises: :class:`CalledProcessError` if the command fails
 136         """
 137
 138         destination = "{}@{}".format(self.user, self.host)
 139         cmd = [
 140             "ssh",
 141             "-i",
 142             os.path.expanduser(self.private_key_path),
 143             "-o",
 144             "StrictHostKeyChecking=no",
 145             "-q",
 146             destination,
 147         ]
 148         cmd.extend([command])
 149         process = await asyncio.create_subprocess_exec(
 150             *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
 151         )
 152         stdout, stderr = await process.communicate()
 153
 154         if process.returncode != 0:
 155             output = stderr.decode("utf-8").strip()
 156             raise CalledProcessError(
 157                 returncode=process.returncode, cmd=cmd, output=output
 158             )
 159         return (stdout.decode("utf-8").strip(), stderr.decode("utf-8").strip())
 160
 161     async def _init_ubuntu_user(self):
 162         """Initialize the ubuntu user.
 163
 164         :return: bool: If the initialization was successful
 165         :raises: :class:`CalledProcessError` if the _ssh command fails
 166         """
 167         retry = 10
 168         attempts = 0
 169         delay = 15
 170         while attempts <= retry:
 171             try:
 172                 attempts += 1
 173                 # Attempt to establish a SSH connection
 174                 stdout, stderr = await self._ssh("sudo -n true")
 175                 break
 176             except CalledProcessError as e:
 177                 self.log.debug(
 178                     "Waiting for VM to boot, sleeping {} seconds".format(delay)
 179                 )
 180                 if attempts > retry:
 181                     raise e
 182                 else:
 183                     await asyncio.sleep(delay)
 184                     # Slowly back off the retry
 185                     delay += 15
 186
 187         # Infer the public key
 188         public_key = None
 189         public_key_path = "{}.pub".format(self.private_key_path)
 190
 191         if not os.path.exists(public_key_path):
 192             raise FileNotFoundError(
 193                 "Public key '{}' doesn't exist.".format(public_key_path)
 194             )
 195
 196         with open(public_key_path, "r") as f:
 197             public_key = f.readline()
 198
 199         script = INITIALIZE_UBUNTU_SCRIPT.format(public_key)
 200
 201         stdout, stderr = await self._run_configure_script(script)
 202
 203         return True
 204
 205     async def _detect_hardware_and_os(self):
 206         """Detect the target hardware capabilities and OS series.
 207
 208         :return: str: A raw string containing OS and hardware information.
 209         """
 210
 211         info = {
 212             "series": "",
 213             "arch": "",
 214             "cpu-cores": "",
 215             "mem": "",
 216         }
 217
 218         stdout, stderr = await self._run_configure_script(DETECTION_SCRIPT)
 219
 220         lines = stdout.split("\n")
 221         info["series"] = lines[0].strip()
 222         info["arch"] = normalize_arch(lines[1].strip())
 223
 224         memKb = re.split(r"\s+", lines[2])[1]
 225
 226         # Convert megabytes -> kilobytes
 227         info["mem"] = round(int(memKb) / 1024)
 228
 229         # Detect available CPUs
 230         recorded = {}
 231         for line in lines[3:]:
 232             physical_id = ""
 233             print(line)
 234
 235             if line.find("physical id") == 0:
 236                 physical_id = line.split(":")[1].strip()
 237             elif line.find("cpu cores") == 0:
 238                 cores = line.split(":")[1].strip()
 239
 240                 if physical_id not in recorded.keys():
 241                     info["cpu-cores"] += cores
 242                     recorded[physical_id] = True
 243
 244         return info
 245
 246     async def provision_machine(self):
 247         """Perform the initial provisioning of the target machine.
 248
 249         :return: bool: The client.AddMachineParams
 250         """
 251         params = client.AddMachineParams()
 252
 253         if await self._init_ubuntu_user():
 254             hw = await self._detect_hardware_and_os()
 255             params.series = hw["series"]
 256             params.instance_id = "manual:{}".format(self.host)
 257             params.nonce = "manual:{}:{}".format(
 258                 self.host,
 259                 str(uuid.uuid4()),
 260             )  # a nop for Juju w/manual machines
 261             params.hardware_characteristics = {
 262                 "arch": hw["arch"],
 263                 "mem": int(hw["mem"]),
 264                 "cpu-cores": int(hw["cpu-cores"]),
 265             }
 266             params.addresses = [{"value": self.host, "type": "ipv4", "scope": "public"}]
 267
 268         return params
 269
 270     async def install_agent(
 271         self, connection, nonce, machine_id, proxy=None, series=None
 272     ):
 273         """
 274         :param object connection: Connection to Juju API
 275         :param str nonce: The nonce machine specification
 276         :param str machine_id: The id assigned to the machine
 277         :param str proxy: IP of the API_PROXY
 278         :param str series: OS name
 279
 280         :return: bool: If the initialization was successful
 281         """
 282         # The path where the Juju agent should be installed.
 283         data_dir = "/var/lib/juju"
 284
 285         # Disabling this prevents `apt-get update` from running initially, so
 286         # charms will fail to deploy
 287         disable_package_commands = False
 288
 289         client_facade = client.ClientFacade.from_connection(connection)
 290         results = await client_facade.ProvisioningScript(
 291             data_dir=data_dir,
 292             disable_package_commands=disable_package_commands,
 293             machine_id=machine_id,
 294             nonce=nonce,
 295         )
 296
 297         """Get the IP of the controller
 298
 299         Parse the provisioning script, looking for the first apiaddress.
 300
 301         Example:
 302             apiaddresses:
 303             - 10.195.8.2:17070
 304             - 127.0.0.1:17070
 305             - '[::1]:17070'
 306         """
 307         try:
 308             # Wait until cloud-init finish
 309             await self._run_configure_script(CLOUD_INIT_WAIT_SCRIPT)
 310         except Exception:
 311             self.log.debug("cloud-init not present in machine {}".format(machine_id))
 312
 313         if proxy:
 314             m = re.search(
 315                 r"apiaddresses:\n- (\d+\.\d+\.\d+\.\d+):17070", results.script
 316             )
 317             apiaddress = m.group(1)
 318
 319             """Add IP Table rule
 320
 321             In order to route the traffic to the private ip of the Juju controller
 322             we use a DNAT rule to tell the machine that the destination for the
 323             private address is the public address of the machine where the Juju
 324             controller is running in LXD. That machine will have a complimentary
 325             iptables rule, routing traffic to the appropriate LXD container.
 326             """
 327
 328             if series and ("centos" in series or "rhel" in series):
 329                 script = IPTABLES_SCRIPT_RHEL.format(apiaddress, proxy)
 330             else:
 331                 script = IPTABLES_SCRIPT.format(apiaddress, proxy)
 332
 333             # Run this in a retry loop, because dpkg may be running and cause the
 334             # script to fail.
 335             retry = 10
 336             attempts = 0
 337             delay = 15
 338
 339             while attempts <= retry:
 340                 try:
 341                     attempts += 1
 342                     stdout, stderr = await self._run_configure_script(script)
 343                     break
 344                 except Exception as e:
 345                     self.log.debug(
 346                         "Waiting for DNAT rules to be applied and saved, "
 347                         "sleeping {} seconds".format(delay)
 348                     )
 349                     if attempts > retry:
 350                         raise e
 351                     else:
 352                         await asyncio.sleep(delay)
 353                         # Slowly back off the retry
 354                         delay += 15
 355
 356         # self.log.debug("Running configure script")
 357         await self._run_configure_script(results.script)
 358         # self.log.debug("Configure script finished")
 359
 360     async def _run_configure_script(self, script, root=True):
 361         """Run the script to install the Juju agent on the target machine.
 362
 363         :param str script: The script to be executed
 364         """
 365         _, tmpFile = tempfile.mkstemp()
 366         with open(tmpFile, "w") as f:
 367             f.write(script)
 368             f.close()
 369
 370         # copy the local copy of the script to the remote machine
 371         await self._scp(tmpFile, tmpFile)
 372
 373         # run the provisioning script
 374         return await self._ssh(
 375             "{} /bin/bash {}".format("sudo" if root else "", tmpFile)
 376         )