1 # Copyright 2019 Canonical Ltd.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
18 from subprocess
import CalledProcessError
23 from juju
.client
import client
24 import n2vc
.exceptions
29 [re
.compile(r
"amd64|x86_64"), "amd64"],
30 [re
.compile(r
"i?[3-9]86"), "i386"],
31 [re
.compile(r
"(arm$)|(armv.*)"), "armhf"],
32 [re
.compile(r
"aarch64"), "arm64"],
33 [re
.compile(r
"ppc64|ppc64el|ppc64le"), "ppc64el"],
34 [re
.compile(r
"s390x?"), "s390x"],
38 def normalize_arch(rawArch
):
39 """Normalize the architecture string."""
41 if arch
[0].match(rawArch
):
45 DETECTION_SCRIPT
= """#!/bin/bash
47 os_id=$(grep '^ID=' /etc/os-release | tr -d '"' | cut -d= -f2)
48 if [ "$os_id" = 'centos' ]; then
49 os_version=$(grep '^VERSION_ID=' /etc/os-release | tr -d '"' | cut -d= -f2)
50 echo "centos$os_version"
55 grep MemTotal /proc/meminfo
59 INITIALIZE_UBUNTU_SCRIPT
= """set -e
60 (id ubuntu &> /dev/null) || useradd -m ubuntu -s /bin/bash
63 echo 'ubuntu ALL=(ALL) NOPASSWD:ALL' > $temp
64 install -m 0440 $temp /etc/sudoers.d/90-juju-ubuntu
66 su ubuntu -c 'install -D -m 0600 /dev/null ~/.ssh/authorized_keys'
67 export authorized_keys="{}"
68 if [ ! -z "$authorized_keys" ]; then
69 su ubuntu -c 'echo $authorized_keys >> ~/.ssh/authorized_keys'
73 IPTABLES_SCRIPT
= """#!/bin/bash
75 [ -v `which netfilter-persistent` ] && apt update \
76 && DEBIAN_FRONTEND=noninteractive apt-get install -yqq iptables-persistent
77 iptables -t nat -A OUTPUT -p tcp -d {} -j DNAT --to-destination {}
78 netfilter-persistent save
82 class AsyncSSHProvisioner
:
83 """Provision a manually created machine via SSH."""
89 def __init__(self
, user
, host
, private_key_path
, log
=None):
92 self
.private_key_path
= private_key_path
93 self
.log
= log
if log
else logging
.getLogger(__name__
)
95 async def _scp(self
, source_file
, destination_file
):
96 """Execute an scp command. Requires a fully qualified source and
99 :param str source_file: Path to the source file
100 :param str destination_file: Path to the destination file
105 os
.path
.expanduser(self
.private_key_path
),
107 "StrictHostKeyChecking=no",
111 destination
= "{}@{}:{}".format(self
.user
, self
.host
, destination_file
)
112 cmd
.extend([source_file
, destination
])
113 process
= await asyncio
.create_subprocess_exec(*cmd
)
115 if process
.returncode
!= 0:
116 raise CalledProcessError(returncode
=process
.returncode
, cmd
=cmd
)
118 async def _ssh(self
, command
):
119 """Run a command remotely via SSH.
121 :param str command: The command to execute
122 :return: tuple: The stdout and stderr of the command execution
123 :raises: :class:`CalledProcessError` if the command fails
126 destination
= "{}@{}".format(self
.user
, self
.host
)
130 os
.path
.expanduser(self
.private_key_path
),
132 "StrictHostKeyChecking=no",
136 cmd
.extend([command
])
137 process
= await asyncio
.create_subprocess_exec(
138 *cmd
, stdout
=asyncio
.subprocess
.PIPE
, stderr
=asyncio
.subprocess
.PIPE
140 stdout
, stderr
= await process
.communicate()
142 if process
.returncode
!= 0:
143 output
= stderr
.decode("utf-8").strip()
144 raise CalledProcessError(
145 returncode
=process
.returncode
, cmd
=cmd
, output
=output
147 return (stdout
.decode("utf-8").strip(), stderr
.decode("utf-8").strip())
149 async def _init_ubuntu_user(self
):
150 """Initialize the ubuntu user.
152 :return: bool: If the initialization was successful
153 :raises: :class:`CalledProcessError` if the _ssh command fails
158 while attempts
<= retry
:
161 # Attempt to establish a SSH connection
162 stdout
, stderr
= await self
._ssh
("sudo -n true")
164 except CalledProcessError
as e
:
166 "Waiting for VM to boot, sleeping {} seconds".format(delay
)
171 await asyncio
.sleep(delay
)
172 # Slowly back off the retry
175 # Infer the public key
177 public_key_path
= "{}.pub".format(self
.private_key_path
)
179 if not os
.path
.exists(public_key_path
):
180 raise FileNotFoundError(
181 "Public key '{}' doesn't exist.".format(public_key_path
)
184 with
open(public_key_path
, "r") as f
:
185 public_key
= f
.readline()
187 script
= INITIALIZE_UBUNTU_SCRIPT
.format(public_key
)
189 stdout
, stderr
= await self
._run
_configure
_script
(script
)
193 async def _detect_hardware_and_os(self
):
194 """Detect the target hardware capabilities and OS series.
196 :return: str: A raw string containing OS and hardware information.
206 stdout
, stderr
= await self
._run
_configure
_script
(DETECTION_SCRIPT
)
208 lines
= stdout
.split("\n")
209 info
["series"] = lines
[0].strip()
210 info
["arch"] = normalize_arch(lines
[1].strip())
212 memKb
= re
.split(r
"\s+", lines
[2])[1]
214 # Convert megabytes -> kilobytes
215 info
["mem"] = round(int(memKb
) / 1024)
217 # Detect available CPUs
219 for line
in lines
[3:]:
223 if line
.find("physical id") == 0:
224 physical_id
= line
.split(":")[1].strip()
225 elif line
.find("cpu cores") == 0:
226 cores
= line
.split(":")[1].strip()
228 if physical_id
not in recorded
.keys():
229 info
["cpu-cores"] += cores
230 recorded
[physical_id
] = True
234 async def provision_machine(self
):
235 """Perform the initial provisioning of the target machine.
237 :return: bool: The client.AddMachineParams
239 params
= client
.AddMachineParams()
241 if await self
._init
_ubuntu
_user
():
242 hw
= await self
._detect
_hardware
_and
_os
()
243 params
.series
= hw
["series"]
244 params
.instance_id
= "manual:{}".format(self
.host
)
245 params
.nonce
= "manual:{}:{}".format(
246 self
.host
, str(uuid
.uuid4()),
247 ) # a nop for Juju w/manual machines
248 params
.hardware_characteristics
= {
250 "mem": int(hw
["mem"]),
251 "cpu-cores": int(hw
["cpu-cores"]),
253 params
.addresses
= [{"value": self
.host
, "type": "ipv4", "scope": "public"}]
257 async def install_agent(self
, connection
, nonce
, machine_id
, api
):
259 :param object connection: Connection to Juju API
260 :param str nonce: The nonce machine specification
261 :param str machine_id: The id assigned to the machine
262 :param str api: IP of the API_PROXY
264 :return: bool: If the initialization was successful
266 # The path where the Juju agent should be installed.
267 data_dir
= "/var/lib/juju"
269 # Disabling this prevents `apt-get update` from running initially, so
270 # charms will fail to deploy
271 disable_package_commands
= False
273 client_facade
= client
.ClientFacade
.from_connection(connection
)
274 results
= await client_facade
.ProvisioningScript(
276 disable_package_commands
=disable_package_commands
,
277 machine_id
=machine_id
,
281 """Get the IP of the controller
283 Parse the provisioning script, looking for the first apiaddress.
291 m
= re
.search(r
"apiaddresses:\n- (\d+\.\d+\.\d+\.\d+):17070", results
.script
)
292 apiaddress
= m
.group(1)
296 In order to route the traffic to the private ip of the Juju controller
297 we use a DNAT rule to tell the machine that the destination for the
298 private address is the public address of the machine where the Juju
299 controller is running in LXD. That machine will have a complimentary
300 iptables rule, routing traffic to the appropriate LXD container.
303 script
= IPTABLES_SCRIPT
.format(apiaddress
, api
)
305 # Run this in a retry loop, because dpkg may be running and cause the
311 while attempts
<= retry
:
314 stdout
, stderr
= await self
._run
_configure
_script
(script
)
316 except Exception as e
:
317 self
.log
.debug("Waiting for dpkg, sleeping {} seconds".format(delay
))
321 await asyncio
.sleep(delay
)
322 # Slowly back off the retry
325 # self.log.debug("Running configure script")
326 await self
._run
_configure
_script
(results
.script
)
327 # self.log.debug("Configure script finished")
329 async def _run_configure_script(self
, script
, root
=True):
330 """Run the script to install the Juju agent on the target machine.
332 :param str script: The script to be executed
334 _
, tmpFile
= tempfile
.mkstemp()
335 with
open(tmpFile
, "w") as f
:
339 # copy the local copy of the script to the remote machine
340 await self
._scp
(tmpFile
, tmpFile
)
342 # run the provisioning script
343 return await self
._ssh
(
344 "{} /bin/bash {}".format("sudo" if root
else "", tmpFile
)
348 class SSHProvisioner
:
349 """Provision a manually created machine via SSH."""
351 def __init__(self
, user
, host
, private_key_path
, log
=None):
355 self
.private_key_path
= private_key_path
360 self
.log
= logging
.getLogger(__name__
)
362 def _get_ssh_client(self
, host
=None, user
=None, private_key_path
=None):
363 """Return a connected Paramiko ssh object.
365 :param str host: The host to connect to.
366 :param str user: The user to connect as.
367 :param str key: The private key to authenticate with.
369 :return: object: A paramiko.SSHClient
370 :raises: :class:`paramiko.ssh_exception.SSHException` if the
380 if not private_key_path
:
381 private_key_path
= self
.private_key_path
383 ssh
= paramiko
.SSHClient()
384 ssh
.set_missing_host_key_policy(paramiko
.AutoAddPolicy())
388 # Read the private key into a paramiko.RSAKey
389 if os
.path
.exists(private_key_path
):
390 with
open(private_key_path
, "r") as f
:
391 pkey
= paramiko
.RSAKey
.from_private_key(f
)
393 #######################################################################
394 # There is a bug in some versions of OpenSSH 4.3 (CentOS/RHEL5) where #
395 # the server may not send the SSH_MSG_USERAUTH_BANNER message except #
396 # when responding to an auth_none request. For example, paramiko will #
397 # attempt to use password authentication when a password is set, but #
398 # the server could deny that, instead requesting keyboard-interactive.#
399 # The hack to workaround this is to attempt a reconnect, which will #
400 # receive the right banner, and authentication can proceed. See the #
401 # following for more info: #
402 # https://github.com/paramiko/paramiko/issues/432 #
403 # https://github.com/paramiko/paramiko/pull/438 #
404 #######################################################################
409 while attempts
<= retry
:
413 # Attempt to establish a SSH connection
420 # look_for_keys=False,
423 except paramiko
.ssh_exception
.SSHException
as e
:
424 if "Error reading SSH protocol banner" == str(e
):
425 # Once more, with feeling
426 ssh
.connect(host
, port
=22, username
=user
, pkey
=pkey
)
428 # Reraise the original exception
429 self
.log
.debug("Unhandled exception caught: {}".format(e
))
431 except Exception as e
:
432 if "Unable to connect to port" in str(e
):
434 "Waiting for VM to boot, sleeping {} seconds".format(delay
)
440 # Slowly back off the retry
447 def _run_command(self
, ssh
, cmd
, pty
=True):
448 """Run a command remotely via SSH.
450 :param object ssh: The SSHClient
451 :param str cmd: The command to execute
452 :param list cmd: The `shlex.split` command to execute
453 :param bool pty: Whether to allocate a pty
455 :return: tuple: The stdout and stderr of the command execution
456 :raises: :class:`CalledProcessError` if the command fails
459 if isinstance(cmd
, str):
460 cmd
= shlex
.split(cmd
)
462 if type(cmd
) is not list:
466 _
, stdout
, stderr
= ssh
.exec_command(cmds
, get_pty
=pty
)
467 retcode
= stdout
.channel
.recv_exit_status()
470 output
= stderr
.read().strip()
471 raise CalledProcessError(returncode
=retcode
, cmd
=cmd
, output
=output
)
473 stdout
.read().decode("utf-8").strip(),
474 stderr
.read().decode("utf-8").strip(),
477 def _init_ubuntu_user(self
):
478 """Initialize the ubuntu user.
480 :return: bool: If the initialization was successful
481 :raises: :class:`paramiko.ssh_exception.AuthenticationException`
482 if the authentication fails
486 # Run w/o allocating a pty, so we fail if sudo prompts for a passwd
487 ssh
= self
._get
_ssh
_client
()
488 self
._run
_command
(ssh
, "sudo -n true", pty
=False)
489 except paramiko
.ssh_exception
.AuthenticationException
:
490 raise n2vc
.exceptions
.AuthenticationFailed(self
.user
)
491 except paramiko
.ssh_exception
.NoValidConnectionsError
:
492 raise n2vc
.exceptions
.NoRouteToHost(self
.host
)
497 # Infer the public key
498 public_key_path
= "{}.pub".format(self
.private_key_path
)
500 if not os
.path
.exists(public_key_path
):
501 raise FileNotFoundError(
502 "Public key '{}' doesn't exist.".format(public_key_path
)
505 with
open(public_key_path
, "r") as f
:
506 public_key
= f
.readline()
508 script
= INITIALIZE_UBUNTU_SCRIPT
.format(public_key
)
511 ssh
= self
._get
_ssh
_client
()
514 ssh
, ["sudo", "/bin/bash -c " + shlex
.quote(script
)], pty
=True
516 except paramiko
.ssh_exception
.AuthenticationException
as e
:
523 def _detect_hardware_and_os(self
, ssh
):
524 """Detect the target hardware capabilities and OS series.
526 :param object ssh: The SSHClient
527 :return: str: A raw string containing OS and hardware information.
537 stdout
, _
= self
._run
_command
(
538 ssh
, ["sudo", "/bin/bash -c " + shlex
.quote(DETECTION_SCRIPT
)], pty
=True,
541 lines
= stdout
.split("\n")
543 # Remove extraneous line if DNS resolution of hostname famils
544 # i.e. sudo: unable to resolve host test-1-mgmtvm-1: Connection timed out
545 if "unable to resolve host" in lines
[0]:
548 info
["series"] = lines
[0].strip()
549 info
["arch"] = normalize_arch(lines
[1].strip())
551 memKb
= re
.split(r
"\s+", lines
[2])[1]
553 # Convert megabytes -> kilobytes
554 info
["mem"] = round(int(memKb
) / 1024)
556 # Detect available CPUs
558 for line
in lines
[3:]:
561 if line
.find("physical id") == 0:
562 physical_id
= line
.split(":")[1].strip()
563 elif line
.find("cpu cores") == 0:
564 cores
= line
.split(":")[1].strip()
566 if physical_id
not in recorded
.keys():
567 info
["cpu-cores"] += cores
568 recorded
[physical_id
] = True
572 def provision_machine(self
):
573 """Perform the initial provisioning of the target machine.
575 :return: bool: The client.AddMachineParams
576 :raises: :class:`paramiko.ssh_exception.AuthenticationException`
579 params
= client
.AddMachineParams()
581 if self
._init
_ubuntu
_user
():
583 ssh
= self
._get
_ssh
_client
()
585 hw
= self
._detect
_hardware
_and
_os
(ssh
)
586 params
.series
= hw
["series"]
587 params
.instance_id
= "manual:{}".format(self
.host
)
588 params
.nonce
= "manual:{}:{}".format(
589 self
.host
, str(uuid
.uuid4()),
590 ) # a nop for Juju w/manual machines
591 params
.hardware_characteristics
= {
593 "mem": int(hw
["mem"]),
594 "cpu-cores": int(hw
["cpu-cores"]),
597 {"value": self
.host
, "type": "ipv4", "scope": "public"}
600 except paramiko
.ssh_exception
.AuthenticationException
as e
:
607 async def install_agent(self
, connection
, nonce
, machine_id
, api
):
609 :param object connection: Connection to Juju API
610 :param str nonce: The nonce machine specification
611 :param str machine_id: The id assigned to the machine
613 :return: bool: If the initialization was successful
615 # The path where the Juju agent should be installed.
616 data_dir
= "/var/lib/juju"
618 # Disabling this prevents `apt-get update` from running initially, so
619 # charms will fail to deploy
620 disable_package_commands
= False
622 client_facade
= client
.ClientFacade
.from_connection(connection
)
623 results
= await client_facade
.ProvisioningScript(
625 disable_package_commands
=disable_package_commands
,
626 machine_id
=machine_id
,
630 """Get the IP of the controller
632 Parse the provisioning script, looking for the first apiaddress.
640 m
= re
.search(r
"apiaddresses:\n- (\d+\.\d+\.\d+\.\d+):17070", results
.script
)
641 apiaddress
= m
.group(1)
645 In order to route the traffic to the private ip of the Juju controller
646 we use a DNAT rule to tell the machine that the destination for the
647 private address is the public address of the machine where the Juju
648 controller is running in LXD. That machine will have a complimentary
649 iptables rule, routing traffic to the appropriate LXD container.
652 script
= IPTABLES_SCRIPT
.format(apiaddress
, api
)
654 # Run this in a retry loop, because dpkg may be running and cause the
660 while attempts
<= retry
:
664 self
._run
_configure
_script
(script
)
666 except Exception as e
:
667 self
.log
.debug("Waiting for dpkg, sleeping {} seconds".format(delay
))
672 # Slowly back off the retry
675 # self.log.debug("Running configure script")
676 self
._run
_configure
_script
(results
.script
)
677 # self.log.debug("Configure script finished")
679 def _run_configure_script(self
, script
: str):
680 """Run the script to install the Juju agent on the target machine.
682 :param str script: The script returned by the ProvisioningScript API
683 :raises: :class:`paramiko.ssh_exception.AuthenticationException`
686 _
, tmpFile
= tempfile
.mkstemp()
687 with
open(tmpFile
, "w") as f
:
691 ssh
= self
._get
_ssh
_client
(user
="ubuntu",)
693 # copy the local copy of the script to the remote machine
694 sftp
= paramiko
.SFTPClient
.from_transport(ssh
.get_transport())
699 # run the provisioning script
701 ssh
, "sudo /bin/bash {}".format(tmpFile
),
704 except paramiko
.ssh_exception
.AuthenticationException
as e
: