Merge "Update deprecated use of yaml.load"
[osm/N2VC.git] / n2vc / provisioner.py
1 # Copyright 2019 Canonical Ltd.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 import logging
15 import os
16 import re
17 import shlex
18 import tempfile
19 import time
20 import uuid
21 from subprocess import CalledProcessError
22
23 import paramiko
24 import n2vc.exceptions
25
26 from juju.client import client
27
28 arches = [
29 [re.compile(r"amd64|x86_64"), "amd64"],
30 [re.compile(r"i?[3-9]86"), "i386"],
31 [re.compile(r"(arm$)|(armv.*)"), "armhf"],
32 [re.compile(r"aarch64"), "arm64"],
33 [re.compile(r"ppc64|ppc64el|ppc64le"), "ppc64el"],
34 [re.compile(r"s390x?"), "s390x"],
35
36 ]
37
38
39 def normalize_arch(rawArch):
40 """Normalize the architecture string."""
41 for arch in arches:
42 if arch[0].match(rawArch):
43 return arch[1]
44
45
46 DETECTION_SCRIPT = """#!/bin/bash
47 set -e
48 os_id=$(grep '^ID=' /etc/os-release | tr -d '"' | cut -d= -f2)
49 if [ "$os_id" = 'centos' ]; then
50 os_version=$(grep '^VERSION_ID=' /etc/os-release | tr -d '"' | cut -d= -f2)
51 echo "centos$os_version"
52 else
53 lsb_release -cs
54 fi
55 uname -m
56 grep MemTotal /proc/meminfo
57 cat /proc/cpuinfo
58 """
59
60 INITIALIZE_UBUNTU_SCRIPT = """set -e
61 (id ubuntu &> /dev/null) || useradd -m ubuntu -s /bin/bash
62 umask 0077
63 temp=$(mktemp)
64 echo 'ubuntu ALL=(ALL) NOPASSWD:ALL' > $temp
65 install -m 0440 $temp /etc/sudoers.d/90-juju-ubuntu
66 rm $temp
67 su ubuntu -c 'install -D -m 0600 /dev/null ~/.ssh/authorized_keys'
68 export authorized_keys="{}"
69 if [ ! -z "$authorized_keys" ]; then
70 su ubuntu -c 'echo $authorized_keys >> ~/.ssh/authorized_keys'
71 fi
72 """
73
74 IPTABLES_SCRIPT = """#!/bin/bash
75 set -e
76 apt-get update
77 DEBIAN_FRONTEND=noninteractive apt-get install -yqq iptables-persistent
78 iptables -t nat -A OUTPUT -p tcp -d {} -j DNAT --to-destination {}
79 netfilter-persistent save
80 """
81
82 class SSHProvisioner:
83 """Provision a manually created machine via SSH."""
84
85 def __init__(self, user, host, private_key_path, log=None):
86
87 self.host = host
88 self.user = user
89 self.private_key_path = private_key_path
90
91 if log:
92 self.log = log
93 else:
94 self.log = logging.getLogger(__name__)
95
96 def _get_ssh_client(self, host=None, user=None, private_key_path=None):
97 """Return a connected Paramiko ssh object.
98
99 :param str host: The host to connect to.
100 :param str user: The user to connect as.
101 :param str key: The private key to authenticate with.
102
103 :return: object: A paramiko.SSHClient
104 :raises: :class:`paramiko.ssh_exception.SSHException` if the
105 connection failed
106 """
107
108 if not host:
109 host = self.host
110
111 if not user:
112 user = self.user
113
114 if not private_key_path:
115 private_key_path = self.private_key_path
116
117 ssh = paramiko.SSHClient()
118 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
119
120 pkey = None
121
122 # Read the private key into a paramiko.RSAKey
123 if os.path.exists(private_key_path):
124 with open(private_key_path, 'r') as f:
125 pkey = paramiko.RSAKey.from_private_key(f)
126
127 #######################################################################
128 # There is a bug in some versions of OpenSSH 4.3 (CentOS/RHEL5) where #
129 # the server may not send the SSH_MSG_USERAUTH_BANNER message except #
130 # when responding to an auth_none request. For example, paramiko will #
131 # attempt to use password authentication when a password is set, but #
132 # the server could deny that, instead requesting keyboard-interactive.#
133 # The hack to workaround this is to attempt a reconnect, which will #
134 # receive the right banner, and authentication can proceed. See the #
135 # following for more info: #
136 # https://github.com/paramiko/paramiko/issues/432 #
137 # https://github.com/paramiko/paramiko/pull/438 #
138 #######################################################################
139
140 retry = 10
141 attempts = 0
142 delay = 15
143 while attempts <= retry:
144 try:
145 attempts += 1
146
147 # Attempt to establish a SSH connection
148 ssh.connect(
149 host,
150 port=22,
151 username=user,
152 pkey=pkey,
153 # allow_agent=False,
154 # look_for_keys=False,
155 )
156 break
157 except paramiko.ssh_exception.SSHException as e:
158 if 'Error reading SSH protocol banner' == str(e):
159 # Once more, with feeling
160 ssh.connect(host, port=22, username=user, pkey=pkey)
161 else:
162 # Reraise the original exception
163 self.log.debug("Unhandled exception caught: {}".format(e))
164 raise e
165 except Exception as e:
166 if 'Unable to connect to port' in str(e):
167 self.log.debug("Waiting for VM to boot, sleeping {} seconds".format(delay))
168 if attempts > retry:
169 raise e
170 else:
171 time.sleep(delay)
172 # Slowly back off the retry
173 delay += 15
174 else:
175 self.log.debug(e)
176 raise e
177 return ssh
178
179 def _run_command(self, ssh, cmd, pty=True):
180 """Run a command remotely via SSH.
181
182 :param object ssh: The SSHClient
183 :param str cmd: The command to execute
184 :param list cmd: The `shlex.split` command to execute
185 :param bool pty: Whether to allocate a pty
186
187 :return: tuple: The stdout and stderr of the command execution
188 :raises: :class:`CalledProcessError` if the command fails
189 """
190
191 if isinstance(cmd, str):
192 cmd = shlex.split(cmd)
193
194 if type(cmd) is not list:
195 cmd = [cmd]
196
197 cmds = ' '.join(cmd)
198 stdin, stdout, stderr = ssh.exec_command(cmds, get_pty=pty)
199 retcode = stdout.channel.recv_exit_status()
200
201 if retcode > 0:
202 output = stderr.read().strip()
203 raise CalledProcessError(returncode=retcode, cmd=cmd,
204 output=output)
205 return (
206 stdout.read().decode('utf-8').strip(),
207 stderr.read().decode('utf-8').strip()
208 )
209
210 def _init_ubuntu_user(self):
211 """Initialize the ubuntu user.
212
213 :return: bool: If the initialization was successful
214 :raises: :class:`paramiko.ssh_exception.AuthenticationException`
215 if the authentication fails
216 """
217 ssh = None
218 try:
219 # Run w/o allocating a pty, so we fail if sudo prompts for a passwd
220 ssh = self._get_ssh_client()
221 stdout, stderr = self._run_command(ssh, "sudo -n true", pty=False)
222 except paramiko.ssh_exception.AuthenticationException:
223 raise n2vc.exceptions.AuthenticationFailed(self.user)
224 except paramiko.ssh_exception.NoValidConnectionsError:
225 raise n2vc.exceptions.NoRouteToHost(self.host)
226 finally:
227 if ssh:
228 ssh.close()
229
230 # Infer the public key
231 public_key = None
232 public_key_path = "{}.pub".format(self.private_key_path)
233
234 if not os.path.exists(public_key_path):
235 raise FileNotFoundError(
236 "Public key '{}' doesn't exist.".format(public_key_path)
237 )
238
239 with open(public_key_path, "r") as f:
240 public_key = f.readline()
241
242 script = INITIALIZE_UBUNTU_SCRIPT.format(public_key)
243
244 try:
245 ssh = self._get_ssh_client()
246
247 self._run_command(
248 ssh,
249 ["sudo", "/bin/bash -c " + shlex.quote(script)],
250 pty=True
251 )
252 except paramiko.ssh_exception.AuthenticationException as e:
253 raise e
254 finally:
255 ssh.close()
256
257 return True
258
259 def _detect_hardware_and_os(self, ssh):
260 """Detect the target hardware capabilities and OS series.
261
262 :param object ssh: The SSHClient
263 :return: str: A raw string containing OS and hardware information.
264 """
265
266 info = {
267 'series': '',
268 'arch': '',
269 'cpu-cores': '',
270 'mem': '',
271 }
272
273 stdout, stderr = self._run_command(
274 ssh,
275 ["sudo", "/bin/bash -c " + shlex.quote(DETECTION_SCRIPT)],
276 pty=True,
277 )
278
279 lines = stdout.split("\n")
280
281 # Remove extraneous line if DNS resolution of hostname famils
282 # i.e. sudo: unable to resolve host test-1-mgmtvm-1: Connection timed out
283 if 'unable to resolve host' in lines[0]:
284 lines = lines[1:]
285
286 info['series'] = lines[0].strip()
287 info['arch'] = normalize_arch(lines[1].strip())
288
289 memKb = re.split(r'\s+', lines[2])[1]
290
291 # Convert megabytes -> kilobytes
292 info['mem'] = round(int(memKb) / 1024)
293
294 # Detect available CPUs
295 recorded = {}
296 for line in lines[3:]:
297 physical_id = ""
298
299 if line.find("physical id") == 0:
300 physical_id = line.split(":")[1].strip()
301 elif line.find("cpu cores") == 0:
302 cores = line.split(":")[1].strip()
303
304 if physical_id not in recorded.keys():
305 info['cpu-cores'] += cores
306 recorded[physical_id] = True
307
308 return info
309
310 def provision_machine(self):
311 """Perform the initial provisioning of the target machine.
312
313 :return: bool: The client.AddMachineParams
314 :raises: :class:`paramiko.ssh_exception.AuthenticationException`
315 if the upload fails
316 """
317 params = client.AddMachineParams()
318
319 if self._init_ubuntu_user():
320 try:
321 ssh = self._get_ssh_client()
322
323 hw = self._detect_hardware_and_os(ssh)
324 params.series = hw['series']
325 params.instance_id = "manual:{}".format(self.host)
326 params.nonce = "manual:{}:{}".format(
327 self.host,
328 str(uuid.uuid4()), # a nop for Juju w/manual machines
329 )
330 params.hardware_characteristics = {
331 'arch': hw['arch'],
332 'mem': int(hw['mem']),
333 'cpu-cores': int(hw['cpu-cores']),
334 }
335 params.addresses = [{
336 'value': self.host,
337 'type': 'ipv4',
338 'scope': 'public',
339 }]
340
341 except paramiko.ssh_exception.AuthenticationException as e:
342 raise e
343 finally:
344 ssh.close()
345
346 return params
347
348 async def install_agent(self, connection, nonce, machine_id, api):
349 """
350 :param object connection: Connection to Juju API
351 :param str nonce: The nonce machine specification
352 :param str machine_id: The id assigned to the machine
353
354 :return: bool: If the initialization was successful
355 """
356 # The path where the Juju agent should be installed.
357 data_dir = "/var/lib/juju"
358
359 # Disabling this prevents `apt-get update` from running initially, so
360 # charms will fail to deploy
361 disable_package_commands = False
362
363 client_facade = client.ClientFacade.from_connection(connection)
364 results = await client_facade.ProvisioningScript(
365 data_dir=data_dir,
366 disable_package_commands=disable_package_commands,
367 machine_id=machine_id,
368 nonce=nonce,
369 )
370
371 """Get the IP of the controller
372
373 Parse the provisioning script, looking for the first apiaddress.
374
375 Example:
376 apiaddresses:
377 - 10.195.8.2:17070
378 - 127.0.0.1:17070
379 - '[::1]:17070'
380 """
381 m = re.search('apiaddresses:\n- (\d+\.\d+\.\d+\.\d+):17070', results.script)
382 apiaddress = m.group(1)
383
384 """Add IP Table rule
385
386 In order to route the traffic to the private ip of the Juju controller
387 we use a DNAT rule to tell the machine that the destination for the
388 private address is the public address of the machine where the Juju
389 controller is running in LXD. That machine will have a complimentary
390 iptables rule, routing traffic to the appropriate LXD container.
391 """
392
393 script = IPTABLES_SCRIPT.format(apiaddress, api)
394
395 # Run this in a retry loop, because dpkg may be running and cause the
396 # script to fail.
397 retry = 10
398 attempts = 0
399 delay = 15
400
401 while attempts <= retry:
402 try:
403 attempts += 1
404
405 self._run_configure_script(script)
406 break
407 except Exception as e:
408 self.log.debug("Waiting for dpkg, sleeping {} seconds".format(delay))
409 if attempts > retry:
410 raise e
411 else:
412 time.sleep(delay)
413 # Slowly back off the retry
414 delay += 15
415
416 # self.log.debug("Running configure script")
417 self._run_configure_script(results.script)
418 # self.log.debug("Configure script finished")
419
420
421
422 def _run_configure_script(self, script: str):
423 """Run the script to install the Juju agent on the target machine.
424
425 :param str script: The script returned by the ProvisioningScript API
426 :raises: :class:`paramiko.ssh_exception.AuthenticationException`
427 if the upload fails
428 """
429 _, tmpFile = tempfile.mkstemp()
430 with open(tmpFile, 'w') as f:
431 f.write(script)
432 try:
433 # get ssh client
434 ssh = self._get_ssh_client(
435 user="ubuntu",
436 )
437
438 # copy the local copy of the script to the remote machine
439 sftp = paramiko.SFTPClient.from_transport(ssh.get_transport())
440 sftp.put(
441 tmpFile,
442 tmpFile,
443 )
444
445 # run the provisioning script
446 stdout, stderr = self._run_command(
447 ssh,
448 "sudo /bin/bash {}".format(tmpFile),
449 )
450
451 except paramiko.ssh_exception.AuthenticationException as e:
452 raise e
453 finally:
454 os.remove(tmpFile)
455 ssh.close()