Disable the check of the release notes
[osm/RO.git] / RO-VIM-gcp / osm_rovim_gcp / vimconn_gcp.py
1 # -*- coding: utf-8 -*-
2 ##
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
13 # under the License.
14 ##
15
16 import logging
17 from os import getenv
18 import random
19 from random import choice as random_choice
20 import time
21
22 from google.oauth2 import service_account
23 import googleapiclient.discovery
24 from osm_ro_plugin import vimconn
25
26 __author__ = "Sergio Gallardo Ruiz"
27 __date__ = "$11-aug-2021 08:30:00$"
28
29
30 if getenv("OSMRO_PDB_DEBUG"):
31 import sys
32
33 print(sys.path)
34 import pdb
35
36 pdb.set_trace()
37
38
39 class vimconnector(vimconn.VimConnector):
40 # Translate Google Cloud provisioning state to OSM provision state
41 # The first three ones are the transitional status once a user initiated action has been requested
42 # Once the operation is complete, it will transition into the states Succeeded or Failed
43 # https://cloud.google.com/compute/docs/instances/instance-life-cycle
44 provision_state2osm = {
45 "PROVISIONING": "BUILD",
46 "REPAIRING": "ERROR",
47 }
48
49 # Translate azure power state to OSM provision state
50 power_state2osm = {
51 "STAGING": "BUILD",
52 "RUNNING": "ACTIVE",
53 "STOPPING": "INACTIVE",
54 "SUSPENDING": "INACTIVE",
55 "SUSPENDED": "INACTIVE",
56 "TERMINATED": "INACTIVE",
57 }
58
59 # If a net or subnet is tried to be deleted and it has an associated resource, the net is marked "to be deleted"
60 # (incluid it's name in the following list). When the instance is deleted, its associated net will be deleted if
61 # they are present in that list
62 nets_to_be_deleted = []
63
64 def __init__(
65 self,
66 uuid,
67 name,
68 tenant_id,
69 tenant_name,
70 url,
71 url_admin=None,
72 user=None,
73 passwd=None,
74 log_level=None,
75 config={},
76 persistent_info={},
77 ):
78 """
79 Constructor of VIM. Raise an exception is some needed parameter is missing, but it must not do any connectivity
80 checking against the VIM
81 Using common constructor parameters.
82 In this case: config must include the following parameters:
83 subscription_id: assigned subscription identifier
84 region_name: current region for network
85 config may also include the following parameter:
86 flavors_pattern: pattern that will be used to select a range of vm sizes, for example
87 "^((?!Standard_B).)*$" will filter out Standard_B range that is cheap but is very overused
88 "^Standard_B" will select a serie B maybe for test environment
89 """
90 vimconn.VimConnector.__init__(
91 self,
92 uuid,
93 name,
94 tenant_id,
95 tenant_name,
96 url,
97 url_admin,
98 user,
99 passwd,
100 log_level,
101 config,
102 persistent_info,
103 )
104
105 # Variable that indicates if client must be reloaded or initialized
106 self.reload_client = False
107
108 # LOGGER
109
110 log_format_simple = (
111 "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
112 )
113 log_format_complete = "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(funcName)s(): %(message)s"
114 log_formatter_simple = logging.Formatter(
115 log_format_simple, datefmt="%Y-%m-%dT%H:%M:%S"
116 )
117 self.handler = logging.StreamHandler()
118 self.handler.setFormatter(log_formatter_simple)
119
120 self.logger = logging.getLogger("ro.vim.gcp")
121 self.logger.addHandler(self.handler)
122 if log_level:
123 self.logger.setLevel(getattr(logging, log_level))
124
125 if self.logger.getEffectiveLevel() == logging.DEBUG:
126 log_formatter = logging.Formatter(
127 log_format_complete, datefmt="%Y-%m-%dT%H:%M:%S"
128 )
129 self.handler.setFormatter(log_formatter)
130
131 self.logger.debug("Google Cloud connection init")
132
133 self.project = tenant_id or tenant_name
134
135 # REGION - Google Cloud considers regions and zones. A specific region can have more than one zone
136 # (for instance: region us-west1 with the zones us-west1-a, us-west1-b and us-west1-c)
137 # So the region name specified in the config will be considered as a specific zone for GC and
138 # the region will be calculated from that without the preffix.
139 if "region_name" in config:
140 self.zone = config.get("region_name")
141 self.region = self.zone.rsplit("-", 1)[0]
142 else:
143 raise vimconn.VimConnException(
144 "Google Cloud region_name is not specified at config"
145 )
146
147 # Credentials
148 self.logger.debug("Config: %s", config)
149 scopes = ["https://www.googleapis.com/auth/cloud-platform"]
150 self.credentials = None
151 if "credentials" in config:
152 self.logger.debug("Setting credentials")
153 # Settings Google Cloud credentials dict
154 credentials_body = config["credentials"]
155 # self.logger.debug("Credentials filtered: %s", credentials_body)
156 credentials = service_account.Credentials.from_service_account_info(
157 credentials_body
158 )
159 if "sa_file" in config:
160 credentials = service_account.Credentials.from_service_account_file(
161 config.get("sa_file"), scopes=scopes
162 )
163 self.logger.debug("Credentials: %s", credentials)
164 # Construct a Resource for interacting with an API.
165 self.credentials = credentials
166 try:
167 self.conn_compute = googleapiclient.discovery.build(
168 "compute", "v1", credentials=credentials
169 )
170 except Exception as e:
171 self._format_vimconn_exception(e)
172 else:
173 raise vimconn.VimConnException(
174 "It is not possible to init GCP with no credentials"
175 )
176
177 def _reload_connection(self):
178 """
179 Called before any operation, checks python Google Cloud clientsself.reload_client
180 """
181 if self.reload_client:
182 self.logger.debug("reloading google cloud client")
183
184 try:
185 # Set to client created
186 self.conn_compute = googleapiclient.discovery.build(
187 "compute", "v1", credentials=self.credentials
188 )
189 except Exception as e:
190 self._format_vimconn_exception(e)
191
192 def _format_vimconn_exception(self, e):
193 """
194 Transforms a generic exception to a vimConnException
195 """
196 self.logger.error("Google Cloud plugin error: {}".format(e))
197 if isinstance(e, vimconn.VimConnException):
198 raise e
199 else:
200 # In case of generic error recreate client
201 self.reload_client = True
202 raise vimconn.VimConnException(type(e).__name__ + ": " + str(e))
203
204 def _wait_for_global_operation(self, operation):
205 """
206 Waits for the end of the specific operation
207 :operation: operation name
208 """
209
210 self.logger.debug("Waiting for operation %s", operation)
211
212 while True:
213 result = (
214 self.conn_compute.globalOperations()
215 .get(project=self.project, operation=operation)
216 .execute()
217 )
218
219 if result["status"] == "DONE":
220 if "error" in result:
221 raise vimconn.VimConnException(result["error"])
222 return result
223
224 time.sleep(1)
225
226 def _wait_for_zone_operation(self, operation):
227 """
228 Waits for the end of the specific operation
229 :operation: operation name
230 """
231
232 self.logger.debug("Waiting for operation %s", operation)
233
234 while True:
235 result = (
236 self.conn_compute.zoneOperations()
237 .get(project=self.project, operation=operation, zone=self.zone)
238 .execute()
239 )
240
241 if result["status"] == "DONE":
242 if "error" in result:
243 raise vimconn.VimConnException(result["error"])
244 return result
245
246 time.sleep(1)
247
248 def _wait_for_region_operation(self, operation):
249 """
250 Waits for the end of the specific operation
251 :operation: operation name
252 """
253
254 self.logger.debug("Waiting for operation %s", operation)
255
256 while True:
257 result = (
258 self.conn_compute.regionOperations()
259 .get(project=self.project, operation=operation, region=self.region)
260 .execute()
261 )
262
263 if result["status"] == "DONE":
264 if "error" in result:
265 raise vimconn.VimConnException(result["error"])
266 return result
267
268 time.sleep(1)
269
270 def new_network(
271 self,
272 net_name,
273 net_type,
274 ip_profile=None,
275 shared=False,
276 provider_network_profile=None,
277 ):
278 """
279 Adds a network to VIM
280 :param net_name: name of the network
281 :param net_type: not used for Google Cloud networks
282 :param ip_profile: not used for Google Cloud networks
283 :param shared: Not allowed for Google Cloud Connector
284 :param provider_network_profile: (optional)
285
286 contains {segmentation-id: vlan, provider-network: vim_netowrk}
287 :return: a tuple with the network identifier and created_items, or raises an exception on error
288 created_items can be None or a dictionary where this method can include key-values that will be passed to
289 the method delete_network. Can be used to store created segments, created l2gw connections, etc.
290 Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
291 as not present.
292 """
293
294 self.logger.debug(
295 "new_network begin: net_name %s net_type %s ip_profile %s shared %s provider_network_profile %s",
296 net_name,
297 net_type,
298 ip_profile,
299 shared,
300 provider_network_profile,
301 )
302 net_name = self._check_vm_name(net_name)
303 net_name = self._randomize_name(net_name)
304 self.logger.debug("create network name %s, ip_profile %s", net_name, ip_profile)
305
306 try:
307 self.logger.debug("creating network_name: {}".format(net_name))
308
309 network = "projects/{}/global/networks/default".format(self.project)
310 subnet_address = ""
311 if ip_profile is not None:
312 if "subnet_address" in ip_profile:
313 subnet_address = ip_profile["subnet_address"]
314 network_body = {
315 "name": str(net_name),
316 "description": net_name,
317 "network": network,
318 "ipCidrRange": subnet_address,
319 # The network is created in AUTO mode (one subnet per region is created)
320 # "autoCreateSubnetworks": True,
321 "autoCreateSubnetworks": False,
322 }
323
324 operation = (
325 self.conn_compute.networks()
326 .insert(project=self.project, body=network_body)
327 .execute()
328 )
329 self._wait_for_global_operation(operation["name"])
330 self.logger.debug("created network_name: {}".format(net_name))
331
332 # Adding firewall rules to allow the traffic in the network:
333 self._create_firewall_rules(net_name)
334
335 # create subnetwork, even if there is no profile
336
337 if not ip_profile:
338 ip_profile = {}
339
340 if not ip_profile.get("subnet_address"):
341 # Fake subnet is required
342 subnet_rand = random.randint(0, 255)
343 ip_profile["subnet_address"] = "192.168.{}.0/24".format(subnet_rand)
344
345 subnet_name = net_name + "-subnet"
346 subnet_id = self._new_subnet(
347 subnet_name, ip_profile, operation["targetLink"]
348 )
349
350 self.logger.debug("new_network Return: subnet_id: %s", subnet_id)
351 return subnet_id
352 except Exception as e:
353 self._format_vimconn_exception(e)
354
355 def _new_subnet(self, subnet_name, ip_profile, network):
356 """
357 Adds a tenant network to VIM. It creates a new subnet at existing base vnet
358 :param net_name: subnet name
359 :param ip_profile:
360 subnet-address: if it is not provided a subnet/24 in the default vnet is created,
361 otherwise it creates a subnet in the indicated address
362 :return: a tuple with the network identifier and created_items, or raises an exception on error
363 """
364 self.logger.debug(
365 "_new_subnet begin: subnet_name %s ip_profile %s network %s",
366 subnet_name,
367 ip_profile,
368 network,
369 )
370 self.logger.debug(
371 "create subnet name %s, ip_profile %s", subnet_name, ip_profile
372 )
373
374 try:
375 self.logger.debug("creating subnet_name: {}".format(subnet_name))
376
377 subnetwork_body = {
378 "name": str(subnet_name),
379 "description": subnet_name,
380 "network": network,
381 "ipCidrRange": ip_profile["subnet_address"],
382 }
383
384 operation = (
385 self.conn_compute.subnetworks()
386 .insert(
387 project=self.project,
388 region=self.region,
389 body=subnetwork_body,
390 )
391 .execute()
392 )
393 self._wait_for_region_operation(operation["name"])
394
395 self.logger.debug("created subnet_name: {}".format(subnet_name))
396
397 self.logger.debug(
398 "_new_subnet Return: (%s,%s)",
399 "regions/%s/subnetworks/%s" % (self.region, subnet_name),
400 None,
401 )
402 return "regions/%s/subnetworks/%s" % (self.region, subnet_name), None
403 except Exception as e:
404 self._format_vimconn_exception(e)
405
406 def get_network_list(self, filter_dict={}):
407 """Obtain tenant networks of VIM
408 Filter_dict can be:
409 name: network name
410 id: network id
411 shared: boolean, not implemented in GC
412 tenant_id: tenant, not used in GC, all networks same tenants
413 admin_state_up: boolean, not implemented in GC
414 status: 'ACTIVE', not implemented in GC #
415 Returns the network list of dictionaries
416 """
417 self.logger.debug("get_network_list begin: filter_dict %s", filter_dict)
418 self.logger.debug(
419 "Getting network (subnetwork) from VIM filter: {}".format(str(filter_dict))
420 )
421
422 try:
423 if self.reload_client:
424 self._reload_connection()
425
426 net_list = []
427
428 request = self.conn_compute.subnetworks().list(
429 project=self.project, region=self.region
430 )
431
432 while request is not None:
433 response = request.execute()
434 self.logger.debug("Network list: %s", response)
435 for net in response["items"]:
436 self.logger.debug("Network in list: {}".format(str(net["name"])))
437 if filter_dict is not None:
438 if "name" in filter_dict.keys():
439 if (
440 filter_dict["name"] == net["name"]
441 or filter_dict["name"] == net["selfLink"]
442 ):
443 self.logger.debug("Network found: %s", net["name"])
444 net_list.append(
445 {
446 "id": str(net["selfLink"]),
447 "name": str(net["name"]),
448 "network": str(net["network"]),
449 }
450 )
451 else:
452 net_list.append(
453 {
454 "id": str(net["selfLink"]),
455 "name": str(net["name"]),
456 "network": str(net["network"]),
457 }
458 )
459 request = self.conn_compute.subnetworks().list_next(
460 previous_request=request, previous_response=response
461 )
462
463 self.logger.debug("get_network_list Return: net_list %s", net_list)
464 return net_list
465
466 except Exception as e:
467 self.logger.error("Error in get_network_list()", exc_info=True)
468 raise vimconn.VimConnException(e)
469
470 def get_network(self, net_id):
471 self.logger.debug("get_network begin: net_id %s", net_id)
472 # res_name = self._get_resource_name_from_resource_id(net_id)
473 self._reload_connection()
474
475 self.logger.debug("Get network: %s", net_id)
476 filter_dict = {"name": net_id}
477 network_list = self.get_network_list(filter_dict)
478 self.logger.debug("Network list: %s", network_list)
479
480 if not network_list:
481 return []
482 else:
483 self.logger.debug("get_network Return: network_list[0] %s", network_list[0])
484 return network_list[0]
485
486 def delete_network(self, net_id, created_items=None):
487 """
488 Removes a tenant network from VIM and its associated elements
489 :param net_id: VIM identifier of the network, provided by method new_network
490 :param created_items: dictionary with extra items to be deleted. provided by method new_network
491 Returns the network identifier or raises an exception upon error or when network is not found
492 """
493
494 self.logger.debug(
495 "delete_network begin: net_id %s created_items %s",
496 net_id,
497 created_items,
498 )
499 self.logger.debug("Deleting network: {}".format(str(net_id)))
500
501 try:
502 net_name = self._get_resource_name_from_resource_id(net_id)
503
504 # Check associated VMs
505 self.conn_compute.instances().list(
506 project=self.project, zone=self.zone
507 ).execute()
508
509 net_id = self.delete_subnet(net_name, created_items)
510
511 self.logger.debug("delete_network Return: net_id %s", net_id)
512 return net_id
513
514 except Exception as e:
515 self.logger.error("Error in delete_network()", exc_info=True)
516 raise vimconn.VimConnException(e)
517
518 def delete_subnet(self, net_id, created_items=None):
519 """
520 Removes a tenant network from VIM and its associated elements
521 :param net_id: VIM identifier of the network, provided by method new_network
522 :param created_items: dictionary with extra items to be deleted. provided by method new_network
523 Returns the network identifier or raises an exception upon error or when network is not found
524 """
525
526 self.logger.debug(
527 "delete_subnet begin: net_id %s created_items %s",
528 net_id,
529 created_items,
530 )
531 self.logger.debug("Deleting subnetwork: {}".format(str(net_id)))
532
533 try:
534 # If the network has no more subnets, it will be deleted too
535 net_info = self.get_network(net_id)
536 # If the subnet is in use by another resource, the deletion will
537 # be retried N times before abort the operation
538 created_items = created_items or {}
539 created_items[net_id] = False
540
541 try:
542 operation = (
543 self.conn_compute.subnetworks()
544 .delete(
545 project=self.project,
546 region=self.region,
547 subnetwork=net_id,
548 )
549 .execute()
550 )
551 self._wait_for_region_operation(operation["name"])
552 if net_id in self.nets_to_be_deleted:
553 self.nets_to_be_deleted.remove(net_id)
554 except Exception as e:
555 if (
556 e.args[0]["status"] == "400"
557 ): # Resource in use, so the net is marked to be deleted
558 self.logger.debug("Subnet still in use")
559 self.nets_to_be_deleted.append(net_id)
560 else:
561 raise vimconn.VimConnException(e)
562
563 self.logger.debug("nets_to_be_deleted: %s", self.nets_to_be_deleted)
564
565 # If the network has no more subnets, it will be deleted too
566 # if "network" in net_info and net_id not in self.nets_to_be_deleted:
567 if "network" in net_info:
568 network_name = self._get_resource_name_from_resource_id(
569 net_info["network"]
570 )
571
572 try:
573 # Deletion of the associated firewall rules:
574 self._delete_firewall_rules(network_name)
575
576 operation = (
577 self.conn_compute.networks()
578 .delete(
579 project=self.project,
580 network=network_name,
581 )
582 .execute()
583 )
584 self._wait_for_global_operation(operation["name"])
585 except Exception as e:
586 self.logger.debug("error deleting associated network %s", e)
587
588 self.logger.debug("delete_subnet Return: net_id %s", net_id)
589 return net_id
590
591 except Exception as e:
592 self.logger.error("Error in delete_network()", exc_info=True)
593 raise vimconn.VimConnException(e)
594
595 def new_flavor(self, flavor_data):
596 """
597 It is not allowed to create new flavors (machine types) in Google Cloud, must always use an existing one
598 """
599 raise vimconn.VimConnNotImplemented(
600 "It is not possible to create new flavors in Google Cloud"
601 )
602
603 def new_tenant(self, tenant_name, tenant_description):
604 """
605 It is not allowed to create new tenants in Google Cloud
606 """
607 raise vimconn.VimConnNotImplemented(
608 "It is not possible to create a TENANT in Google Cloud"
609 )
610
611 def get_flavor(self, flavor_id):
612 """
613 Obtains the flavor_data from the flavor_id/machine type id
614 """
615 self.logger.debug("get_flavor begin: flavor_id %s", flavor_id)
616
617 try:
618 response = (
619 self.conn_compute.machineTypes()
620 .get(project=self.project, zone=self.zone, machineType=flavor_id)
621 .execute()
622 )
623 flavor_data = response
624 self.logger.debug("Machine type data: %s", flavor_data)
625
626 if flavor_data:
627 flavor = {
628 "id": flavor_data["id"],
629 "name": flavor_id,
630 "id_complete": flavor_data["selfLink"],
631 "ram": flavor_data["memoryMb"],
632 "vcpus": flavor_data["guestCpus"],
633 "disk": flavor_data["maximumPersistentDisksSizeGb"],
634 }
635
636 self.logger.debug("get_flavor Return: flavor %s", flavor)
637 return flavor
638 else:
639 raise vimconn.VimConnNotFoundException(
640 "flavor '{}' not found".format(flavor_id)
641 )
642 except Exception as e:
643 self._format_vimconn_exception(e)
644
645 # Google Cloud VM names can not have some special characters
646 def _check_vm_name(self, vm_name):
647 """
648 Checks vm name, in case the vm has not allowed characters they are removed, not error raised
649 Only lowercase and hyphens are allowed
650 """
651 chars_not_allowed_list = "~!@#$%^&*()=+_[]{}|;:<>/?."
652
653 # First: the VM name max length is 64 characters
654 vm_name_aux = vm_name[:62]
655
656 # Second: replace not allowed characters
657 for elem in chars_not_allowed_list:
658 # Check if string is in the main string
659 if elem in vm_name_aux:
660 # self.logger.debug("Dentro del IF")
661 # Replace the string
662 vm_name_aux = vm_name_aux.replace(elem, "-")
663
664 return vm_name_aux.lower()
665
666 def get_flavor_id_from_data(self, flavor_dict):
667 self.logger.debug("get_flavor_id_from_data begin: flavor_dict %s", flavor_dict)
668 filter_dict = flavor_dict or {}
669
670 try:
671 response = (
672 self.conn_compute.machineTypes()
673 .list(project=self.project, zone=self.zone)
674 .execute()
675 )
676 machine_types_list = response["items"]
677 # self.logger.debug("List of machine types: %s", machine_types_list)
678
679 cpus = filter_dict.get("vcpus") or 0
680 memMB = filter_dict.get("ram") or 0
681 # Workaround (it should be 0)
682 numberInterfaces = len(filter_dict.get("interfaces", [])) or 4
683
684 # Filter
685 filtered_machines = []
686 for machine_type in machine_types_list:
687 if (
688 machine_type["guestCpus"] >= cpus
689 and machine_type["memoryMb"] >= memMB
690 # In Google Cloud the number of virtual network interfaces scales with
691 # the number of virtual CPUs with a minimum of 2 and a maximum of 8:
692 # https://cloud.google.com/vpc/docs/create-use-multiple-interfaces#max-interfaces
693 and machine_type["guestCpus"] >= numberInterfaces
694 ):
695 filtered_machines.append(machine_type)
696
697 # self.logger.debug("Filtered machines: %s", filtered_machines)
698
699 # Sort
700 listedFilteredMachines = sorted(
701 filtered_machines,
702 key=lambda k: (
703 int(k["guestCpus"]),
704 float(k["memoryMb"]),
705 int(k["maximumPersistentDisksSizeGb"]),
706 k["name"],
707 ),
708 )
709 # self.logger.debug("Sorted filtered machines: %s", listedFilteredMachines)
710
711 if listedFilteredMachines:
712 self.logger.debug(
713 "get_flavor_id_from_data Return: listedFilteredMachines[0][name] %s",
714 listedFilteredMachines[0]["name"],
715 )
716 return listedFilteredMachines[0]["name"]
717
718 raise vimconn.VimConnNotFoundException(
719 "Cannot find any flavor matching '{}'".format(str(flavor_dict))
720 )
721
722 except Exception as e:
723 self._format_vimconn_exception(e)
724
725 def delete_flavor(self, flavor_id):
726 raise vimconn.VimConnNotImplemented(
727 "It is not possible to delete a flavor in Google Cloud"
728 )
729
730 def delete_tenant(self, tenant_id):
731 raise vimconn.VimConnNotImplemented(
732 "It is not possible to delete a TENANT in Google Cloud"
733 )
734
735 def new_image(self, image_dict):
736 """
737 This function comes from the early days when we though the image could be embedded in the package.
738 Unless OSM manages VM images E2E from NBI to RO, this function does not make sense to be implemented.
739 """
740 raise vimconn.VimConnNotImplemented("Not implemented")
741
742 def get_image_id_from_path(self, path):
743 """
744 This function comes from the early days when we though the image could be embedded in the package.
745 Unless OSM manages VM images E2E from NBI to RO, this function does not make sense to be implemented.
746 """
747 raise vimconn.VimConnNotImplemented("Not implemented")
748
749 def get_image_list(self, filter_dict={}):
750 """Obtain tenant images from VIM
751 Filter_dict can be:
752 name: image name with the format: image project:image family:image version
753 If some part of the name is provide ex: publisher:offer it will search all availables skus and version
754 for the provided publisher and offer
755 id: image uuid, currently not supported for azure
756 Returns the image list of dictionaries:
757 [{<the fields at Filter_dict plus some VIM specific>}, ...]
758 List can be empty
759 """
760 self.logger.debug("get_image_list begin: filter_dict %s", filter_dict)
761
762 try:
763 image_list = []
764 # Get image id from parameter image_id:
765 # <image Project>:image-family:<family> => Latest version of the family
766 # <image Project>:image:<image> => Specific image
767 # <image Project>:<image> => Specific image
768
769 image_info = filter_dict["name"].split(":")
770 image_project = image_info[0]
771 if len(image_info) == 2:
772 image_type = "image"
773 image_item = image_info[1]
774 if len(image_info) == 3:
775 image_type = image_info[1]
776 image_item = image_info[2]
777 else:
778 raise vimconn.VimConnNotFoundException("Wrong format for image")
779
780 image_response = {}
781 if image_type == "image-family":
782 image_response = (
783 self.conn_compute.images()
784 .getFromFamily(project=image_project, family=image_item)
785 .execute()
786 )
787 elif image_type == "image":
788 image_response = (
789 self.conn_compute.images()
790 .get(project=image_project, image=image_item)
791 .execute()
792 )
793 else:
794 raise vimconn.VimConnNotFoundException("Wrong format for image")
795 image_list.append(
796 {
797 "id": "projects/%s/global/images/%s"
798 % (image_project, image_response["name"]),
799 "name": ":".join(
800 [image_project, image_item, image_response["name"]]
801 ),
802 }
803 )
804
805 self.logger.debug("get_image_list Return: image_list %s", image_list)
806 return image_list
807
808 except Exception as e:
809 self._format_vimconn_exception(e)
810
811 def delete_image(self, image_id):
812 raise vimconn.VimConnNotImplemented("Not implemented")
813
814 def action_vminstance(self, vm_id, action_dict, created_items={}):
815 """Send and action over a VM instance from VIM
816 Returns the vm_id if the action was successfully sent to the VIM
817 """
818 raise vimconn.VimConnNotImplemented("Not necessary")
819
820 def _randomize_name(self, name):
821 """Adds a random string to allow requests with the same VM name
822 Returns the name with an additional random string (if the total size is bigger
823 than 62 the original name will be truncated)
824 """
825 random_name = name
826
827 while True:
828 try:
829 random_name = (
830 name[:49]
831 + "-"
832 + "".join(random_choice("0123456789abcdef") for _ in range(12))
833 )
834 self.conn_compute.instances().get(
835 project=self.project, zone=self.zone, instance=random_name
836 ).execute()
837 # If no exception is arisen, the random name exists for an instance,
838 # so a new random name must be generated
839
840 except Exception as e:
841 if e.args[0]["status"] == "404":
842 self.logger.debug("New random name: %s", random_name)
843 break
844 else:
845 self.logger.error(
846 "Exception generating random name (%s) for the instance", name
847 )
848 self._format_vimconn_exception(e)
849
850 return random_name
851
852 def new_vminstance(
853 self,
854 name,
855 description,
856 start,
857 image_id=None, # <image project>:(image|image-family):<image/family id>
858 flavor_id=None,
859 affinity_group_list=None,
860 net_list=None,
861 cloud_config=None,
862 disk_list=None,
863 availability_zone_index=None,
864 availability_zone_list=None,
865 ):
866 self.logger.debug(
867 "new_vminstance begin: name: %s, image_id: %s, flavor_id: %s, net_list: %s, cloud_config: %s, "
868 "disk_list: %s, availability_zone_index: %s, availability_zone_list: %s",
869 name,
870 image_id,
871 flavor_id,
872 net_list,
873 cloud_config,
874 disk_list,
875 availability_zone_index,
876 availability_zone_list,
877 )
878
879 if self.reload_client:
880 self._reload_connection()
881
882 # Validate input data is valid
883 # # First of all, the name must be adapted because Google Cloud only allows names consist of
884 # lowercase letters (a-z), numbers and hyphens (?:[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?)
885 vm_name = self._check_vm_name(name)
886 vm_name = self._randomize_name(vm_name)
887 vm_id = None
888
889 # At least one network must be provided
890 if not net_list:
891 raise vimconn.VimConnException(
892 "At least one net must be provided to create a new VM"
893 )
894
895 try:
896 created_items = {}
897 metadata = self._build_metadata(vm_name, cloud_config)
898
899 # Building network interfaces list
900 network_interfaces = []
901 for net in net_list:
902 net_iface = {}
903 if not net.get("net_id"):
904 if not net.get("name"):
905 continue
906 else:
907 net_iface["subnetwork"] = (
908 "regions/%s/subnetworks/" % self.region + net.get("name")
909 )
910 else:
911 net_iface["subnetwork"] = net.get("net_id")
912 if net.get("ip_address"):
913 net_iface["networkIP"] = net.get("ip_address")
914
915 # In order to get an external IP address, the key "accessConfigs" must be used
916 # in the interace. It has to be of type "ONE_TO_ONE_NAT" and name "External NAT"
917 if net.get("floating_ip", False) or (
918 net["use"] == "mgmt" and self.config.get("use_floating_ip")
919 ):
920 net_iface["accessConfigs"] = [
921 {"type": "ONE_TO_ONE_NAT", "name": "External NAT"}
922 ]
923
924 network_interfaces.append(net_iface)
925
926 self.logger.debug("Network interfaces: %s", network_interfaces)
927
928 self.logger.debug("Source image: %s", image_id)
929
930 vm_parameters = {
931 "name": vm_name,
932 "machineType": self.get_flavor(flavor_id)["id_complete"],
933 # Specify the boot disk and the image to use as a source.
934 "disks": [
935 {
936 "boot": True,
937 "autoDelete": True,
938 "initializeParams": {
939 "sourceImage": image_id,
940 },
941 }
942 ],
943 # Specify the network interfaces
944 "networkInterfaces": network_interfaces,
945 "metadata": metadata,
946 }
947
948 response = (
949 self.conn_compute.instances()
950 .insert(project=self.project, zone=self.zone, body=vm_parameters)
951 .execute()
952 )
953 self._wait_for_zone_operation(response["name"])
954
955 # The created instance info is obtained to get the name of the generated network interfaces (nic0, nic1...)
956 response = (
957 self.conn_compute.instances()
958 .get(project=self.project, zone=self.zone, instance=vm_name)
959 .execute()
960 )
961 self.logger.debug("instance get: %s", response)
962 vm_id = response["name"]
963
964 # The generated network interfaces in the instance are include in net_list:
965 for _, net in enumerate(net_list):
966 for net_ifaces in response["networkInterfaces"]:
967 network_id = ""
968 if "net_id" in net:
969 network_id = self._get_resource_name_from_resource_id(
970 net["net_id"]
971 )
972 else:
973 network_id = self._get_resource_name_from_resource_id(
974 net["name"]
975 )
976 if network_id == self._get_resource_name_from_resource_id(
977 net_ifaces["subnetwork"]
978 ):
979 net["vim_id"] = net_ifaces["name"]
980
981 self.logger.debug(
982 "new_vminstance Return: (name %s, created_items %s)",
983 vm_name,
984 created_items,
985 )
986 return vm_name, created_items
987
988 except Exception as e:
989 # Rollback vm creacion
990 if vm_id is not None:
991 try:
992 self.logger.debug("exception creating vm try to rollback")
993 self.delete_vminstance(vm_id, created_items)
994 except Exception as e2:
995 self.logger.error("new_vminstance rollback fail {}".format(e2))
996
997 else:
998 self.logger.debug(
999 "Exception creating new vminstance: %s", e, exc_info=True
1000 )
1001 self._format_vimconn_exception(e)
1002
1003 def _build_metadata(self, vm_name, cloud_config):
1004 # initial metadata
1005 metadata = {}
1006 metadata["items"] = []
1007
1008 # if there is a cloud-init load it
1009 if cloud_config:
1010 self.logger.debug("cloud config: %s", cloud_config)
1011 _, userdata = self._create_user_data(cloud_config)
1012 metadata["items"].append({"key": "user-data", "value": userdata})
1013
1014 # either password of ssh-keys are required
1015 # we will always use ssh-keys, in case it is not available we will generate it
1016 self.logger.debug("metadata: %s", metadata)
1017
1018 return metadata
1019
1020 def get_vminstance(self, vm_id):
1021 """
1022 Obtaing the vm instance data from v_id
1023 """
1024 self.logger.debug("get_vminstance begin: vm_id %s", vm_id)
1025 self._reload_connection()
1026 response = {}
1027 try:
1028 response = (
1029 self.conn_compute.instances()
1030 .get(project=self.project, zone=self.zone, instance=vm_id)
1031 .execute()
1032 )
1033 # vm = response["source"]
1034 except Exception as e:
1035 self._format_vimconn_exception(e)
1036
1037 self.logger.debug("get_vminstance Return: response %s", response)
1038 return response
1039
1040 def delete_vminstance(self, vm_id, created_items=None, volumes_to_hold=None):
1041 """Deletes a vm instance from the vim."""
1042 self.logger.debug(
1043 "delete_vminstance begin: vm_id %s created_items %s",
1044 vm_id,
1045 created_items,
1046 )
1047 if self.reload_client:
1048 self._reload_connection()
1049
1050 created_items = created_items or {}
1051 try:
1052 vm = self.get_vminstance(vm_id)
1053
1054 operation = (
1055 self.conn_compute.instances()
1056 .delete(project=self.project, zone=self.zone, instance=vm_id)
1057 .execute()
1058 )
1059 self._wait_for_zone_operation(operation["name"])
1060
1061 # The associated subnets must be checked if they are marked to be deleted
1062 for netIface in vm["networkInterfaces"]:
1063 if (
1064 self._get_resource_name_from_resource_id(netIface["subnetwork"])
1065 in self.nets_to_be_deleted
1066 ):
1067 self._get_resource_name_from_resource_id(
1068 self.delete_network(netIface["subnetwork"])
1069 )
1070
1071 self.logger.debug("delete_vminstance end")
1072
1073 except Exception as e:
1074 # The VM can be deleted previously during network deletion
1075 if e.args[0]["status"] == "404":
1076 self.logger.debug("The VM doesn't exist or has been deleted")
1077 else:
1078 self._format_vimconn_exception(e)
1079
1080 def _get_resource_name_from_resource_id(self, resource_id):
1081 """
1082 Obtains resource_name from the google cloud complete identifier: resource_name will always be last item
1083 """
1084 self.logger.debug(
1085 "_get_resource_name_from_resource_id begin: resource_id %s",
1086 resource_id,
1087 )
1088 try:
1089 resource = str(resource_id.split("/")[-1])
1090
1091 self.logger.debug(
1092 "_get_resource_name_from_resource_id Return: resource %s",
1093 resource,
1094 )
1095 return resource
1096 except Exception as e:
1097 raise vimconn.VimConnException(
1098 "Unable to get resource name from resource_id '{}' Error: '{}'".format(
1099 resource_id, e
1100 )
1101 )
1102
1103 def _get_id_from_image(self, image):
1104 """
1105 Obtains image_id from the google cloud complete image identifier: image_id will be the last five items
1106 """
1107 self.logger.debug(f"_get_id_from_image begin: image {image}")
1108 try:
1109 image_id = "/".join(image.split("/")[-5:])
1110 self.logger.debug(f"_get_id_from_image Return: image_id {image_id}")
1111 return image_id
1112 except Exception as e:
1113 raise vimconn.VimConnException(
1114 f"Unable to get image_id from image '{image}' Error: '{e}'"
1115 )
1116
1117 def refresh_nets_status(self, net_list):
1118 """Get the status of the networks
1119 Params: the list of network identifiers
1120 Returns a dictionary with:
1121 net_id: #VIM id of this network
1122 status: #Mandatory. Text with one of:
1123 # DELETED (not found at vim)
1124 # VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1125 # OTHER (Vim reported other status not understood)
1126 # ERROR (VIM indicates an ERROR status)
1127 # ACTIVE, INACTIVE, DOWN (admin down),
1128 # BUILD (on building process)
1129 #
1130 error_msg: #Text with VIM error message, if any. Or the VIM connection ERROR
1131 vim_info: #Text with plain information obtained from vim (yaml.safe_dump)
1132 """
1133 self.logger.debug("refresh_nets_status begin: net_list %s", net_list)
1134 out_nets = {}
1135 self._reload_connection()
1136
1137 for net_id in net_list:
1138 try:
1139 resName = self._get_resource_name_from_resource_id(net_id)
1140
1141 net = (
1142 self.conn_compute.subnetworks()
1143 .get(project=self.project, region=self.region, subnetwork=resName)
1144 .execute()
1145 )
1146 self.logger.debug("get subnetwork: %s", net)
1147
1148 out_nets[net_id] = {
1149 "status": "ACTIVE", # Google Cloud does not provide the status in subnetworks getting
1150 "vim_info": str(net),
1151 }
1152 except vimconn.VimConnNotFoundException as e:
1153 self.logger.error(
1154 "VimConnNotFoundException %s when searching subnet", e
1155 )
1156 out_nets[net_id] = {
1157 "status": "DELETED",
1158 "error_msg": str(e),
1159 }
1160 except Exception as e:
1161 self.logger.error(
1162 "Exception %s when searching subnet", e, exc_info=True
1163 )
1164 out_nets[net_id] = {
1165 "status": "VIM_ERROR",
1166 "error_msg": str(e),
1167 }
1168
1169 self.logger.debug("refresh_nets_status Return: out_nets %s", out_nets)
1170 return out_nets
1171
1172 def refresh_vms_status(self, vm_list):
1173 """Get the status of the virtual machines and their interfaces/ports
1174 Params: the list of VM identifiers
1175 Returns a dictionary with:
1176 vm_id: # VIM id of this Virtual Machine
1177 status: # Mandatory. Text with one of:
1178 # DELETED (not found at vim)
1179 # VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1180 # OTHER (Vim reported other status not understood)
1181 # ERROR (VIM indicates an ERROR status)
1182 # ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
1183 # BUILD (on building process), ERROR
1184 # ACTIVE:NoMgmtIP (Active but none of its interfaces has an IP address
1185 # (ACTIVE:NoMgmtIP is not returned for Azure)
1186 #
1187 error_msg: #Text with VIM error message, if any. Or the VIM connection ERROR
1188 vim_info: #Text with plain information obtained from vim (yaml.safe_dump)
1189 interfaces: list with interface info. Each item a dictionary with:
1190 vim_interface_id - The ID of the interface
1191 mac_address - The MAC address of the interface.
1192 ip_address - The IP address of the interface within the subnet.
1193 """
1194 self.logger.debug("refresh_vms_status begin: vm_list %s", vm_list)
1195 out_vms = {}
1196 self._reload_connection()
1197
1198 search_vm_list = vm_list or {}
1199
1200 for vm_id in search_vm_list:
1201 out_vm = {}
1202 try:
1203 res_name = self._get_resource_name_from_resource_id(vm_id)
1204
1205 vm = (
1206 self.conn_compute.instances()
1207 .get(project=self.project, zone=self.zone, instance=res_name)
1208 .execute()
1209 )
1210
1211 disk_source = vm["disks"][0]["source"]
1212 self.logger.debug("getting disk information")
1213 disk = (
1214 self.conn_compute.disks()
1215 .get(
1216 project=self.project,
1217 zone=self.zone,
1218 disk=self._get_resource_name_from_resource_id(disk_source),
1219 )
1220 .execute()
1221 )
1222 image = {}
1223 if disk is not None:
1224 self.logger.debug(f"disk: {disk}")
1225 image = {
1226 "id": self._get_id_from_image(disk["sourceImage"]),
1227 "source": disk_source,
1228 }
1229
1230 vim_info = {
1231 "id": vm_id,
1232 "name": vm["name"],
1233 "creationTimestamp": vm["creationTimestamp"],
1234 "lastStartTimestamp": vm["lastStartTimestamp"],
1235 "vm_id": vm["id"],
1236 "kind": vm["kind"],
1237 "cpuPlatform": vm["cpuPlatform"],
1238 "zone": self._get_resource_name_from_resource_id(vm["zone"]),
1239 "machineType": vm["machineType"],
1240 "flavor": {
1241 "id": self._get_resource_name_from_resource_id(
1242 vm["machineType"]
1243 )
1244 },
1245 "image": image,
1246 }
1247 out_vm["vim_info"] = str(vim_info)
1248 out_vm["status"] = self.provision_state2osm.get(vm["status"], "OTHER")
1249
1250 # In Google Cloud the there is no difference between provision or power status,
1251 # so if provision status method doesn't return a specific state (OTHER), the
1252 # power method is called
1253 if out_vm["status"] == "OTHER":
1254 out_vm["status"] = self.power_state2osm.get(vm["status"], "OTHER")
1255
1256 network_interfaces = vm["networkInterfaces"]
1257 out_vm["interfaces"] = self._get_vm_interfaces_status(
1258 vm_id, network_interfaces
1259 )
1260 except Exception as e:
1261 self.logger.error("Exception %s refreshing vm_status", e, exc_info=True)
1262 out_vm["status"] = "VIM_ERROR"
1263 out_vm["error_msg"] = str(e)
1264 out_vm["vim_info"] = None
1265
1266 out_vms[vm_id] = out_vm
1267
1268 self.logger.debug("refresh_vms_status Return: out_vms %s", out_vms)
1269 return out_vms
1270
1271 def _get_vm_interfaces_status(self, vm_id, interfaces):
1272 """
1273 Gets the interfaces detail for a vm
1274 :param interfaces: List of interfaces.
1275 :return: Dictionary with list of interfaces including, vim_interface_id, mac_address and ip_address
1276 """
1277 self.logger.debug(
1278 "_get_vm_interfaces_status begin: vm_id %s interfaces %s",
1279 vm_id,
1280 interfaces,
1281 )
1282 try:
1283 interface_list = []
1284 for network_interface in interfaces:
1285 interface_dict = {}
1286 interface_dict["vim_interface_id"] = network_interface["name"]
1287 interface_dict["vim_net_id"] = network_interface["subnetwork"]
1288
1289 ips = []
1290 ips.append(network_interface["networkIP"])
1291 interface_dict["ip_address"] = ";".join(ips)
1292 interface_list.append(interface_dict)
1293
1294 self.logger.debug(
1295 "_get_vm_interfaces_status Return: interface_list %s",
1296 interface_list,
1297 )
1298 return interface_list
1299 except Exception as e:
1300 self.logger.error(
1301 "Exception %s obtaining interface data for vm: %s",
1302 e,
1303 vm_id,
1304 exc_info=True,
1305 )
1306 self._format_vimconn_exception(e)
1307
1308 def _create_firewall_rules(self, network):
1309 """
1310 Creates the necessary firewall rules to allow the traffic in the network
1311 (https://cloud.google.com/vpc/docs/firewalls)
1312 :param network.
1313 :return: a list with the names of the firewall rules
1314 """
1315 self.logger.debug("_create_firewall_rules begin: network %s", network)
1316 try:
1317 rules_list = []
1318
1319 # Adding firewall rule to allow http:
1320 self.logger.debug("creating firewall rule to allow http")
1321 firewall_rule_body = {
1322 "name": "fw-rule-http-" + network,
1323 "network": "global/networks/" + network,
1324 "allowed": [{"IPProtocol": "tcp", "ports": ["80"]}],
1325 }
1326 self.conn_compute.firewalls().insert(
1327 project=self.project, body=firewall_rule_body
1328 ).execute()
1329
1330 # Adding firewall rule to allow ssh:
1331 self.logger.debug("creating firewall rule to allow ssh")
1332 firewall_rule_body = {
1333 "name": "fw-rule-ssh-" + network,
1334 "network": "global/networks/" + network,
1335 "allowed": [{"IPProtocol": "tcp", "ports": ["22"]}],
1336 }
1337 self.conn_compute.firewalls().insert(
1338 project=self.project, body=firewall_rule_body
1339 ).execute()
1340
1341 # Adding firewall rule to allow ping:
1342 self.logger.debug("creating firewall rule to allow ping")
1343 firewall_rule_body = {
1344 "name": "fw-rule-icmp-" + network,
1345 "network": "global/networks/" + network,
1346 "allowed": [{"IPProtocol": "icmp"}],
1347 }
1348 self.conn_compute.firewalls().insert(
1349 project=self.project, body=firewall_rule_body
1350 ).execute()
1351
1352 # Adding firewall rule to allow internal:
1353 self.logger.debug("creating firewall rule to allow internal")
1354 firewall_rule_body = {
1355 "name": "fw-rule-internal-" + network,
1356 "network": "global/networks/" + network,
1357 "allowed": [
1358 {"IPProtocol": "tcp", "ports": ["0-65535"]},
1359 {"IPProtocol": "udp", "ports": ["0-65535"]},
1360 {"IPProtocol": "icmp"},
1361 ],
1362 }
1363 self.conn_compute.firewalls().insert(
1364 project=self.project, body=firewall_rule_body
1365 ).execute()
1366
1367 # Adding firewall rule to allow microk8s:
1368 self.logger.debug("creating firewall rule to allow microk8s")
1369 firewall_rule_body = {
1370 "name": "fw-rule-microk8s-" + network,
1371 "network": "global/networks/" + network,
1372 "allowed": [{"IPProtocol": "tcp", "ports": ["16443"]}],
1373 }
1374 self.conn_compute.firewalls().insert(
1375 project=self.project, body=firewall_rule_body
1376 ).execute()
1377
1378 # Adding firewall rule to allow rdp:
1379 self.logger.debug("creating firewall rule to allow rdp")
1380 firewall_rule_body = {
1381 "name": "fw-rule-rdp-" + network,
1382 "network": "global/networks/" + network,
1383 "allowed": [{"IPProtocol": "tcp", "ports": ["3389"]}],
1384 }
1385 self.conn_compute.firewalls().insert(
1386 project=self.project, body=firewall_rule_body
1387 ).execute()
1388
1389 # Adding firewall rule to allow osm:
1390 self.logger.debug("creating firewall rule to allow osm")
1391 firewall_rule_body = {
1392 "name": "fw-rule-osm-" + network,
1393 "network": "global/networks/" + network,
1394 "allowed": [{"IPProtocol": "tcp", "ports": ["9001", "9999"]}],
1395 }
1396 self.conn_compute.firewalls().insert(
1397 project=self.project, body=firewall_rule_body
1398 ).execute()
1399
1400 self.logger.debug(
1401 "_create_firewall_rules Return: list_rules %s", rules_list
1402 )
1403 return rules_list
1404 except Exception as e:
1405 self.logger.error(
1406 "Unable to create google cloud firewall rules for network '{}'".format(
1407 network
1408 )
1409 )
1410 self._format_vimconn_exception(e)
1411
1412 def _delete_firewall_rules(self, network):
1413 """
1414 Deletes the associated firewall rules to the network
1415 :param network.
1416 :return: a list with the names of the firewall rules
1417 """
1418 self.logger.debug("_delete_firewall_rules begin: network %s", network)
1419 try:
1420 rules_list = []
1421
1422 rules_list = (
1423 self.conn_compute.firewalls().list(project=self.project).execute()
1424 )
1425 for item in rules_list["items"]:
1426 if network == self._get_resource_name_from_resource_id(item["network"]):
1427 self.conn_compute.firewalls().delete(
1428 project=self.project, firewall=item["name"]
1429 ).execute()
1430
1431 self.logger.debug("_delete_firewall_rules Return: list_rules %s", 0)
1432 return rules_list
1433 except Exception as e:
1434 self.logger.error(
1435 "Unable to delete google cloud firewall rules for network '{}'".format(
1436 network
1437 )
1438 )
1439 self._format_vimconn_exception(e)
1440
1441 def migrate_instance(self, vm_id, compute_host=None):
1442 """
1443 Migrate a vdu
1444 param:
1445 vm_id: ID of an instance
1446 compute_host: Host to migrate the vdu to
1447 """
1448 # TODO: Add support for migration
1449 raise vimconn.VimConnNotImplemented("Not implemented")
1450
1451 def resize_instance(self, vm_id, flavor_id=None):
1452 """
1453 resize a vdu
1454 param:
1455 vm_id: ID of an instance
1456 flavor_id: flavor_id to resize the vdu to
1457 """
1458 # TODO: Add support for resize
1459 raise vimconn.VimConnNotImplemented("Not implemented")