7e7f606d25a351b785d60dcb1be0e0d6a6263118
[osm/RO.git] / RO-VIM-gcp / osm_rovim_gcp / vimconn_gcp.py
1 # -*- coding: utf-8 -*-
2 ##
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
13 # under the License.
14 ##
15
16 import logging
17 from os import getenv
18 import random
19 from random import choice as random_choice
20 import time
21
22 from google.oauth2 import service_account
23 import googleapiclient.discovery
24 from osm_ro_plugin import vimconn
25
26 __author__ = "Sergio Gallardo Ruiz"
27 __date__ = "$11-aug-2021 08:30:00$"
28
29
30 if getenv("OSMRO_PDB_DEBUG"):
31 import sys
32
33 print(sys.path)
34 import pdb
35
36 pdb.set_trace()
37
38
39 class vimconnector(vimconn.VimConnector):
40 # Translate Google Cloud provisioning state to OSM provision state
41 # The first three ones are the transitional status once a user initiated action has been requested
42 # Once the operation is complete, it will transition into the states Succeeded or Failed
43 # https://cloud.google.com/compute/docs/instances/instance-life-cycle
44 provision_state2osm = {
45 "PROVISIONING": "BUILD",
46 "REPAIRING": "ERROR",
47 }
48
49 # Translate azure power state to OSM provision state
50 power_state2osm = {
51 "STAGING": "BUILD",
52 "RUNNING": "ACTIVE",
53 "STOPPING": "INACTIVE",
54 "SUSPENDING": "INACTIVE",
55 "SUSPENDED": "INACTIVE",
56 "TERMINATED": "INACTIVE",
57 }
58
59 # If a net or subnet is tried to be deleted and it has an associated resource, the net is marked "to be deleted"
60 # (incluid it's name in the following list). When the instance is deleted, its associated net will be deleted if
61 # they are present in that list
62 nets_to_be_deleted = []
63
64 def __init__(
65 self,
66 uuid,
67 name,
68 tenant_id,
69 tenant_name,
70 url,
71 url_admin=None,
72 user=None,
73 passwd=None,
74 log_level=None,
75 config={},
76 persistent_info={},
77 ):
78 """
79 Constructor of VIM. Raise an exception is some needed parameter is missing, but it must not do any connectivity
80 checking against the VIM
81 Using common constructor parameters.
82 In this case: config must include the following parameters:
83 subscription_id: assigned subscription identifier
84 region_name: current region for network
85 config may also include the following parameter:
86 flavors_pattern: pattern that will be used to select a range of vm sizes, for example
87 "^((?!Standard_B).)*$" will filter out Standard_B range that is cheap but is very overused
88 "^Standard_B" will select a serie B maybe for test environment
89 """
90 vimconn.VimConnector.__init__(
91 self,
92 uuid,
93 name,
94 tenant_id,
95 tenant_name,
96 url,
97 url_admin,
98 user,
99 passwd,
100 log_level,
101 config,
102 persistent_info,
103 )
104
105 # Variable that indicates if client must be reloaded or initialized
106 self.reload_client = False
107
108 # LOGGER
109
110 log_format_simple = (
111 "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
112 )
113 log_format_complete = "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(funcName)s(): %(message)s"
114 log_formatter_simple = logging.Formatter(
115 log_format_simple, datefmt="%Y-%m-%dT%H:%M:%S"
116 )
117 self.handler = logging.StreamHandler()
118 self.handler.setFormatter(log_formatter_simple)
119
120 self.logger = logging.getLogger("ro.vim.gcp")
121 self.logger.addHandler(self.handler)
122 if log_level:
123 self.logger.setLevel(getattr(logging, log_level))
124
125 if self.logger.getEffectiveLevel() == logging.DEBUG:
126 log_formatter = logging.Formatter(
127 log_format_complete, datefmt="%Y-%m-%dT%H:%M:%S"
128 )
129 self.handler.setFormatter(log_formatter)
130
131 self.logger.debug("Google Cloud connection init")
132
133 self.project = tenant_id or tenant_name
134
135 # REGION - Google Cloud considers regions and zones. A specific region can have more than one zone
136 # (for instance: region us-west1 with the zones us-west1-a, us-west1-b and us-west1-c)
137 # So the region name specified in the config will be considered as a specific zone for GC and
138 # the region will be calculated from that without the preffix.
139 if "region_name" in config:
140 self.zone = config.get("region_name")
141 self.region = self.zone.rsplit("-", 1)[0]
142 else:
143 raise vimconn.VimConnException(
144 "Google Cloud region_name is not specified at config"
145 )
146
147 # Credentials
148 self.logger.debug("Config: %s", config)
149 scopes = ["https://www.googleapis.com/auth/cloud-platform"]
150 self.credentials = None
151 if "credentials" in config:
152 self.logger.debug("Setting credentials")
153 # Settings Google Cloud credentials dict
154 credentials_body = config["credentials"]
155 # self.logger.debug("Credentials filtered: %s", credentials_body)
156 credentials = service_account.Credentials.from_service_account_info(
157 credentials_body
158 )
159 if "sa_file" in config:
160 credentials = service_account.Credentials.from_service_account_file(
161 config.get("sa_file"), scopes=scopes
162 )
163 self.logger.debug("Credentials: %s", credentials)
164 # Construct a Resource for interacting with an API.
165 self.credentials = credentials
166 try:
167 self.conn_compute = googleapiclient.discovery.build(
168 "compute", "v1", credentials=credentials
169 )
170 except Exception as e:
171 self._format_vimconn_exception(e)
172 else:
173 raise vimconn.VimConnException(
174 "It is not possible to init GCP with no credentials"
175 )
176
177 def _reload_connection(self):
178 """
179 Called before any operation, checks python Google Cloud clientsself.reload_client
180 """
181 if self.reload_client:
182 self.logger.debug("reloading google cloud client")
183
184 try:
185 # Set to client created
186 self.conn_compute = googleapiclient.discovery.build("compute", "v1")
187 except Exception as e:
188 self._format_vimconn_exception(e)
189
190 def _format_vimconn_exception(self, e):
191 """
192 Transforms a generic exception to a vimConnException
193 """
194 self.logger.error("Google Cloud plugin error: {}".format(e))
195 if isinstance(e, vimconn.VimConnException):
196 raise e
197 else:
198 # In case of generic error recreate client
199 self.reload_client = True
200 raise vimconn.VimConnException(type(e).__name__ + ": " + str(e))
201
202 def _wait_for_global_operation(self, operation):
203 """
204 Waits for the end of the specific operation
205 :operation: operation name
206 """
207
208 self.logger.debug("Waiting for operation %s", operation)
209
210 while True:
211 result = (
212 self.conn_compute.globalOperations()
213 .get(project=self.project, operation=operation)
214 .execute()
215 )
216
217 if result["status"] == "DONE":
218 if "error" in result:
219 raise vimconn.VimConnException(result["error"])
220 return result
221
222 time.sleep(1)
223
224 def _wait_for_zone_operation(self, operation):
225 """
226 Waits for the end of the specific operation
227 :operation: operation name
228 """
229
230 self.logger.debug("Waiting for operation %s", operation)
231
232 while True:
233 result = (
234 self.conn_compute.zoneOperations()
235 .get(project=self.project, operation=operation, zone=self.zone)
236 .execute()
237 )
238
239 if result["status"] == "DONE":
240 if "error" in result:
241 raise vimconn.VimConnException(result["error"])
242 return result
243
244 time.sleep(1)
245
246 def _wait_for_region_operation(self, operation):
247 """
248 Waits for the end of the specific operation
249 :operation: operation name
250 """
251
252 self.logger.debug("Waiting for operation %s", operation)
253
254 while True:
255 result = (
256 self.conn_compute.regionOperations()
257 .get(project=self.project, operation=operation, region=self.region)
258 .execute()
259 )
260
261 if result["status"] == "DONE":
262 if "error" in result:
263 raise vimconn.VimConnException(result["error"])
264 return result
265
266 time.sleep(1)
267
268 def new_network(
269 self,
270 net_name,
271 net_type,
272 ip_profile=None,
273 shared=False,
274 provider_network_profile=None,
275 ):
276 """
277 Adds a network to VIM
278 :param net_name: name of the network
279 :param net_type: not used for Google Cloud networks
280 :param ip_profile: not used for Google Cloud networks
281 :param shared: Not allowed for Google Cloud Connector
282 :param provider_network_profile: (optional)
283
284 contains {segmentation-id: vlan, provider-network: vim_netowrk}
285 :return: a tuple with the network identifier and created_items, or raises an exception on error
286 created_items can be None or a dictionary where this method can include key-values that will be passed to
287 the method delete_network. Can be used to store created segments, created l2gw connections, etc.
288 Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
289 as not present.
290 """
291
292 self.logger.debug(
293 "new_network begin: net_name %s net_type %s ip_profile %s shared %s provider_network_profile %s",
294 net_name,
295 net_type,
296 ip_profile,
297 shared,
298 provider_network_profile,
299 )
300 net_name = self._check_vm_name(net_name)
301 net_name = self._randomize_name(net_name)
302 self.logger.debug("create network name %s, ip_profile %s", net_name, ip_profile)
303
304 try:
305 self.logger.debug("creating network_name: {}".format(net_name))
306
307 network = "projects/{}/global/networks/default".format(self.project)
308 subnet_address = ""
309 if ip_profile is not None:
310 if "subnet_address" in ip_profile:
311 subnet_address = ip_profile["subnet_address"]
312 network_body = {
313 "name": str(net_name),
314 "description": net_name,
315 "network": network,
316 "ipCidrRange": subnet_address,
317 # The network is created in AUTO mode (one subnet per region is created)
318 # "autoCreateSubnetworks": True,
319 "autoCreateSubnetworks": False,
320 }
321
322 operation = (
323 self.conn_compute.networks()
324 .insert(project=self.project, body=network_body)
325 .execute()
326 )
327 self._wait_for_global_operation(operation["name"])
328 self.logger.debug("created network_name: {}".format(net_name))
329
330 # Adding firewall rules to allow the traffic in the network:
331 self._create_firewall_rules(net_name)
332
333 # create subnetwork, even if there is no profile
334
335 if not ip_profile:
336 ip_profile = {}
337
338 if not ip_profile.get("subnet_address"):
339 # Fake subnet is required
340 subnet_rand = random.randint(0, 255)
341 ip_profile["subnet_address"] = "192.168.{}.0/24".format(subnet_rand)
342
343 subnet_name = net_name + "-subnet"
344 subnet_id = self._new_subnet(
345 subnet_name, ip_profile, operation["targetLink"]
346 )
347
348 self.logger.debug("new_network Return: subnet_id: %s", subnet_id)
349 return subnet_id
350 except Exception as e:
351 self._format_vimconn_exception(e)
352
353 def _new_subnet(self, subnet_name, ip_profile, network):
354 """
355 Adds a tenant network to VIM. It creates a new subnet at existing base vnet
356 :param net_name: subnet name
357 :param ip_profile:
358 subnet-address: if it is not provided a subnet/24 in the default vnet is created,
359 otherwise it creates a subnet in the indicated address
360 :return: a tuple with the network identifier and created_items, or raises an exception on error
361 """
362 self.logger.debug(
363 "_new_subnet begin: subnet_name %s ip_profile %s network %s",
364 subnet_name,
365 ip_profile,
366 network,
367 )
368 self.logger.debug(
369 "create subnet name %s, ip_profile %s", subnet_name, ip_profile
370 )
371
372 try:
373 self.logger.debug("creating subnet_name: {}".format(subnet_name))
374
375 subnetwork_body = {
376 "name": str(subnet_name),
377 "description": subnet_name,
378 "network": network,
379 "ipCidrRange": ip_profile["subnet_address"],
380 }
381
382 operation = (
383 self.conn_compute.subnetworks()
384 .insert(
385 project=self.project,
386 region=self.region,
387 body=subnetwork_body,
388 )
389 .execute()
390 )
391 self._wait_for_region_operation(operation["name"])
392
393 self.logger.debug("created subnet_name: {}".format(subnet_name))
394
395 self.logger.debug(
396 "_new_subnet Return: (%s,%s)",
397 "regions/%s/subnetworks/%s" % (self.region, subnet_name),
398 None,
399 )
400 return "regions/%s/subnetworks/%s" % (self.region, subnet_name), None
401 except Exception as e:
402 self._format_vimconn_exception(e)
403
404 def get_network_list(self, filter_dict={}):
405 """Obtain tenant networks of VIM
406 Filter_dict can be:
407 name: network name
408 id: network id
409 shared: boolean, not implemented in GC
410 tenant_id: tenant, not used in GC, all networks same tenants
411 admin_state_up: boolean, not implemented in GC
412 status: 'ACTIVE', not implemented in GC #
413 Returns the network list of dictionaries
414 """
415 self.logger.debug("get_network_list begin: filter_dict %s", filter_dict)
416 self.logger.debug(
417 "Getting network (subnetwork) from VIM filter: {}".format(str(filter_dict))
418 )
419
420 try:
421 if self.reload_client:
422 self._reload_connection()
423
424 net_list = []
425
426 request = self.conn_compute.subnetworks().list(
427 project=self.project, region=self.region
428 )
429
430 while request is not None:
431 response = request.execute()
432 self.logger.debug("Network list: %s", response)
433 for net in response["items"]:
434 self.logger.debug("Network in list: {}".format(str(net["name"])))
435 if filter_dict is not None:
436 if "name" in filter_dict.keys():
437 if (
438 filter_dict["name"] == net["name"]
439 or filter_dict["name"] == net["selfLink"]
440 ):
441 self.logger.debug("Network found: %s", net["name"])
442 net_list.append(
443 {
444 "id": str(net["selfLink"]),
445 "name": str(net["name"]),
446 "network": str(net["network"]),
447 }
448 )
449 else:
450 net_list.append(
451 {
452 "id": str(net["selfLink"]),
453 "name": str(net["name"]),
454 "network": str(net["network"]),
455 }
456 )
457 request = self.conn_compute.subnetworks().list_next(
458 previous_request=request, previous_response=response
459 )
460
461 self.logger.debug("get_network_list Return: net_list %s", net_list)
462 return net_list
463
464 except Exception as e:
465 self.logger.error("Error in get_network_list()", exc_info=True)
466 raise vimconn.VimConnException(e)
467
468 def get_network(self, net_id):
469 self.logger.debug("get_network begin: net_id %s", net_id)
470 # res_name = self._get_resource_name_from_resource_id(net_id)
471 self._reload_connection()
472
473 self.logger.debug("Get network: %s", net_id)
474 filter_dict = {"name": net_id}
475 network_list = self.get_network_list(filter_dict)
476 self.logger.debug("Network list: %s", network_list)
477
478 if not network_list:
479 return []
480 else:
481 self.logger.debug("get_network Return: network_list[0] %s", network_list[0])
482 return network_list[0]
483
484 def delete_network(self, net_id, created_items=None):
485 """
486 Removes a tenant network from VIM and its associated elements
487 :param net_id: VIM identifier of the network, provided by method new_network
488 :param created_items: dictionary with extra items to be deleted. provided by method new_network
489 Returns the network identifier or raises an exception upon error or when network is not found
490 """
491
492 self.logger.debug(
493 "delete_network begin: net_id %s created_items %s",
494 net_id,
495 created_items,
496 )
497 self.logger.debug("Deleting network: {}".format(str(net_id)))
498
499 try:
500 net_name = self._get_resource_name_from_resource_id(net_id)
501
502 # Check associated VMs
503 self.conn_compute.instances().list(
504 project=self.project, zone=self.zone
505 ).execute()
506
507 net_id = self.delete_subnet(net_name, created_items)
508
509 self.logger.debug("delete_network Return: net_id %s", net_id)
510 return net_id
511
512 except Exception as e:
513 self.logger.error("Error in delete_network()", exc_info=True)
514 raise vimconn.VimConnException(e)
515
516 def delete_subnet(self, net_id, created_items=None):
517 """
518 Removes a tenant network from VIM and its associated elements
519 :param net_id: VIM identifier of the network, provided by method new_network
520 :param created_items: dictionary with extra items to be deleted. provided by method new_network
521 Returns the network identifier or raises an exception upon error or when network is not found
522 """
523
524 self.logger.debug(
525 "delete_subnet begin: net_id %s created_items %s",
526 net_id,
527 created_items,
528 )
529 self.logger.debug("Deleting subnetwork: {}".format(str(net_id)))
530
531 try:
532 # If the network has no more subnets, it will be deleted too
533 net_info = self.get_network(net_id)
534 # If the subnet is in use by another resource, the deletion will
535 # be retried N times before abort the operation
536 created_items = created_items or {}
537 created_items[net_id] = False
538
539 try:
540 operation = (
541 self.conn_compute.subnetworks()
542 .delete(
543 project=self.project,
544 region=self.region,
545 subnetwork=net_id,
546 )
547 .execute()
548 )
549 self._wait_for_region_operation(operation["name"])
550 if net_id in self.nets_to_be_deleted:
551 self.nets_to_be_deleted.remove(net_id)
552 except Exception as e:
553 if (
554 e.args[0]["status"] == "400"
555 ): # Resource in use, so the net is marked to be deleted
556 self.logger.debug("Subnet still in use")
557 self.nets_to_be_deleted.append(net_id)
558 else:
559 raise vimconn.VimConnException(e)
560
561 self.logger.debug("nets_to_be_deleted: %s", self.nets_to_be_deleted)
562
563 # If the network has no more subnets, it will be deleted too
564 # if "network" in net_info and net_id not in self.nets_to_be_deleted:
565 if "network" in net_info:
566 network_name = self._get_resource_name_from_resource_id(
567 net_info["network"]
568 )
569
570 try:
571 # Deletion of the associated firewall rules:
572 self._delete_firewall_rules(network_name)
573
574 operation = (
575 self.conn_compute.networks()
576 .delete(
577 project=self.project,
578 network=network_name,
579 )
580 .execute()
581 )
582 self._wait_for_global_operation(operation["name"])
583 except Exception as e:
584 self.logger.debug("error deleting associated network %s", e)
585
586 self.logger.debug("delete_subnet Return: net_id %s", net_id)
587 return net_id
588
589 except Exception as e:
590 self.logger.error("Error in delete_network()", exc_info=True)
591 raise vimconn.VimConnException(e)
592
593 def new_flavor(self, flavor_data):
594 """
595 It is not allowed to create new flavors (machine types) in Google Cloud, must always use an existing one
596 """
597 raise vimconn.VimConnNotImplemented(
598 "It is not possible to create new flavors in Google Cloud"
599 )
600
601 def new_tenant(self, tenant_name, tenant_description):
602 """
603 It is not allowed to create new tenants in Google Cloud
604 """
605 raise vimconn.VimConnNotImplemented(
606 "It is not possible to create a TENANT in Google Cloud"
607 )
608
609 def get_flavor(self, flavor_id):
610 """
611 Obtains the flavor_data from the flavor_id/machine type id
612 """
613 self.logger.debug("get_flavor begin: flavor_id %s", flavor_id)
614
615 try:
616 response = (
617 self.conn_compute.machineTypes()
618 .get(project=self.project, zone=self.zone, machineType=flavor_id)
619 .execute()
620 )
621 flavor_data = response
622 self.logger.debug("Machine type data: %s", flavor_data)
623
624 if flavor_data:
625 flavor = {
626 "id": flavor_data["id"],
627 "name": flavor_id,
628 "id_complete": flavor_data["selfLink"],
629 "ram": flavor_data["memoryMb"],
630 "vcpus": flavor_data["guestCpus"],
631 "disk": flavor_data["maximumPersistentDisksSizeGb"],
632 }
633
634 self.logger.debug("get_flavor Return: flavor %s", flavor)
635 return flavor
636 else:
637 raise vimconn.VimConnNotFoundException(
638 "flavor '{}' not found".format(flavor_id)
639 )
640 except Exception as e:
641 self._format_vimconn_exception(e)
642
643 # Google Cloud VM names can not have some special characters
644 def _check_vm_name(self, vm_name):
645 """
646 Checks vm name, in case the vm has not allowed characters they are removed, not error raised
647 Only lowercase and hyphens are allowed
648 """
649 chars_not_allowed_list = "~!@#$%^&*()=+_[]{}|;:<>/?."
650
651 # First: the VM name max length is 64 characters
652 vm_name_aux = vm_name[:62]
653
654 # Second: replace not allowed characters
655 for elem in chars_not_allowed_list:
656 # Check if string is in the main string
657 if elem in vm_name_aux:
658 # self.logger.debug("Dentro del IF")
659 # Replace the string
660 vm_name_aux = vm_name_aux.replace(elem, "-")
661
662 return vm_name_aux.lower()
663
664 def get_flavor_id_from_data(self, flavor_dict):
665 self.logger.debug("get_flavor_id_from_data begin: flavor_dict %s", flavor_dict)
666 filter_dict = flavor_dict or {}
667
668 try:
669 response = (
670 self.conn_compute.machineTypes()
671 .list(project=self.project, zone=self.zone)
672 .execute()
673 )
674 machine_types_list = response["items"]
675 # self.logger.debug("List of machine types: %s", machine_types_list)
676
677 cpus = filter_dict.get("vcpus") or 0
678 memMB = filter_dict.get("ram") or 0
679 # Workaround (it should be 0)
680 numberInterfaces = len(filter_dict.get("interfaces", [])) or 4
681
682 # Filter
683 filtered_machines = []
684 for machine_type in machine_types_list:
685 if (
686 machine_type["guestCpus"] >= cpus
687 and machine_type["memoryMb"] >= memMB
688 # In Google Cloud the number of virtual network interfaces scales with
689 # the number of virtual CPUs with a minimum of 2 and a maximum of 8:
690 # https://cloud.google.com/vpc/docs/create-use-multiple-interfaces#max-interfaces
691 and machine_type["guestCpus"] >= numberInterfaces
692 ):
693 filtered_machines.append(machine_type)
694
695 # self.logger.debug("Filtered machines: %s", filtered_machines)
696
697 # Sort
698 listedFilteredMachines = sorted(
699 filtered_machines,
700 key=lambda k: (
701 int(k["guestCpus"]),
702 float(k["memoryMb"]),
703 int(k["maximumPersistentDisksSizeGb"]),
704 k["name"],
705 ),
706 )
707 # self.logger.debug("Sorted filtered machines: %s", listedFilteredMachines)
708
709 if listedFilteredMachines:
710 self.logger.debug(
711 "get_flavor_id_from_data Return: listedFilteredMachines[0][name] %s",
712 listedFilteredMachines[0]["name"],
713 )
714 return listedFilteredMachines[0]["name"]
715
716 raise vimconn.VimConnNotFoundException(
717 "Cannot find any flavor matching '{}'".format(str(flavor_dict))
718 )
719
720 except Exception as e:
721 self._format_vimconn_exception(e)
722
723 def delete_flavor(self, flavor_id):
724 raise vimconn.VimConnNotImplemented(
725 "It is not possible to delete a flavor in Google Cloud"
726 )
727
728 def delete_tenant(self, tenant_id):
729 raise vimconn.VimConnNotImplemented(
730 "It is not possible to delete a TENANT in Google Cloud"
731 )
732
733 def new_image(self, image_dict):
734 """
735 This function comes from the early days when we though the image could be embedded in the package.
736 Unless OSM manages VM images E2E from NBI to RO, this function does not make sense to be implemented.
737 """
738 raise vimconn.VimConnNotImplemented("Not implemented")
739
740 def get_image_id_from_path(self, path):
741 """
742 This function comes from the early days when we though the image could be embedded in the package.
743 Unless OSM manages VM images E2E from NBI to RO, this function does not make sense to be implemented.
744 """
745 raise vimconn.VimConnNotImplemented("Not implemented")
746
747 def get_image_list(self, filter_dict={}):
748 """Obtain tenant images from VIM
749 Filter_dict can be:
750 name: image name with the format: image project:image family:image version
751 If some part of the name is provide ex: publisher:offer it will search all availables skus and version
752 for the provided publisher and offer
753 id: image uuid, currently not supported for azure
754 Returns the image list of dictionaries:
755 [{<the fields at Filter_dict plus some VIM specific>}, ...]
756 List can be empty
757 """
758 self.logger.debug("get_image_list begin: filter_dict %s", filter_dict)
759
760 try:
761 image_list = []
762 # Get image id from parameter image_id:
763 # <image Project>:image-family:<family> => Latest version of the family
764 # <image Project>:image:<image> => Specific image
765 # <image Project>:<image> => Specific image
766
767 image_info = filter_dict["name"].split(":")
768 image_project = image_info[0]
769 if len(image_info) == 2:
770 image_type = "image"
771 image_item = image_info[1]
772 if len(image_info) == 3:
773 image_type = image_info[1]
774 image_item = image_info[2]
775 else:
776 raise vimconn.VimConnNotFoundException("Wrong format for image")
777
778 image_response = {}
779 if image_type == "image-family":
780 image_response = (
781 self.conn_compute.images()
782 .getFromFamily(project=image_project, family=image_item)
783 .execute()
784 )
785 elif image_type == "image":
786 image_response = (
787 self.conn_compute.images()
788 .get(project=image_project, image=image_item)
789 .execute()
790 )
791 else:
792 raise vimconn.VimConnNotFoundException("Wrong format for image")
793 image_list.append(
794 {
795 "id": "projects/%s/global/images/%s"
796 % (image_project, image_response["name"]),
797 "name": ":".join(
798 [image_project, image_item, image_response["name"]]
799 ),
800 }
801 )
802
803 self.logger.debug("get_image_list Return: image_list %s", image_list)
804 return image_list
805
806 except Exception as e:
807 self._format_vimconn_exception(e)
808
809 def delete_image(self, image_id):
810 raise vimconn.VimConnNotImplemented("Not implemented")
811
812 def action_vminstance(self, vm_id, action_dict, created_items={}):
813 """Send and action over a VM instance from VIM
814 Returns the vm_id if the action was successfully sent to the VIM
815 """
816 raise vimconn.VimConnNotImplemented("Not necessary")
817
818 def _randomize_name(self, name):
819 """Adds a random string to allow requests with the same VM name
820 Returns the name with an additional random string (if the total size is bigger
821 than 62 the original name will be truncated)
822 """
823 random_name = name
824
825 while True:
826 try:
827 random_name = (
828 name[:49]
829 + "-"
830 + "".join(random_choice("0123456789abcdef") for _ in range(12))
831 )
832 self.conn_compute.instances().get(
833 project=self.project, zone=self.zone, instance=random_name
834 ).execute()
835 # If no exception is arisen, the random name exists for an instance,
836 # so a new random name must be generated
837
838 except Exception as e:
839 if e.args[0]["status"] == "404":
840 self.logger.debug("New random name: %s", random_name)
841 break
842 else:
843 self.logger.error(
844 "Exception generating random name (%s) for the instance", name
845 )
846 self._format_vimconn_exception(e)
847
848 return random_name
849
850 def new_vminstance(
851 self,
852 name,
853 description,
854 start,
855 image_id=None, # <image project>:(image|image-family):<image/family id>
856 flavor_id=None,
857 affinity_group_list=None,
858 net_list=None,
859 cloud_config=None,
860 disk_list=None,
861 availability_zone_index=None,
862 availability_zone_list=None,
863 ):
864 self.logger.debug(
865 "new_vminstance begin: name: %s, image_id: %s, flavor_id: %s, net_list: %s, cloud_config: %s, "
866 "disk_list: %s, availability_zone_index: %s, availability_zone_list: %s",
867 name,
868 image_id,
869 flavor_id,
870 net_list,
871 cloud_config,
872 disk_list,
873 availability_zone_index,
874 availability_zone_list,
875 )
876
877 if self.reload_client:
878 self._reload_connection()
879
880 # Validate input data is valid
881 # # First of all, the name must be adapted because Google Cloud only allows names consist of
882 # lowercase letters (a-z), numbers and hyphens (?:[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?)
883 vm_name = self._check_vm_name(name)
884 vm_name = self._randomize_name(vm_name)
885 vm_id = None
886
887 # At least one network must be provided
888 if not net_list:
889 raise vimconn.VimConnException(
890 "At least one net must be provided to create a new VM"
891 )
892
893 try:
894 created_items = {}
895 metadata = self._build_metadata(vm_name, cloud_config)
896
897 # Building network interfaces list
898 network_interfaces = []
899 for net in net_list:
900 net_iface = {}
901 if not net.get("net_id"):
902 if not net.get("name"):
903 continue
904 else:
905 net_iface[
906 "subnetwork"
907 ] = "regions/%s/subnetworks/" % self.region + net.get("name")
908 else:
909 net_iface["subnetwork"] = net.get("net_id")
910 # In order to get an external IP address, the key "accessConfigs" must be used
911 # in the interace. It has to be of type "ONE_TO_ONE_NAT" and name "External NAT"
912 if net.get("floating_ip", False) or (
913 net["use"] == "mgmt" and self.config.get("use_floating_ip")
914 ):
915 net_iface["accessConfigs"] = [
916 {"type": "ONE_TO_ONE_NAT", "name": "External NAT"}
917 ]
918
919 network_interfaces.append(net_iface)
920
921 self.logger.debug("Network interfaces: %s", network_interfaces)
922
923 self.logger.debug("Source image: %s", image_id)
924
925 vm_parameters = {
926 "name": vm_name,
927 "machineType": self.get_flavor(flavor_id)["id_complete"],
928 # Specify the boot disk and the image to use as a source.
929 "disks": [
930 {
931 "boot": True,
932 "autoDelete": True,
933 "initializeParams": {
934 "sourceImage": image_id,
935 },
936 }
937 ],
938 # Specify the network interfaces
939 "networkInterfaces": network_interfaces,
940 "metadata": metadata,
941 }
942
943 response = (
944 self.conn_compute.instances()
945 .insert(project=self.project, zone=self.zone, body=vm_parameters)
946 .execute()
947 )
948 self._wait_for_zone_operation(response["name"])
949
950 # The created instance info is obtained to get the name of the generated network interfaces (nic0, nic1...)
951 response = (
952 self.conn_compute.instances()
953 .get(project=self.project, zone=self.zone, instance=vm_name)
954 .execute()
955 )
956 self.logger.debug("instance get: %s", response)
957 vm_id = response["name"]
958
959 # The generated network interfaces in the instance are include in net_list:
960 for _, net in enumerate(net_list):
961 for net_ifaces in response["networkInterfaces"]:
962 network_id = ""
963 if "net_id" in net:
964 network_id = self._get_resource_name_from_resource_id(
965 net["net_id"]
966 )
967 else:
968 network_id = self._get_resource_name_from_resource_id(
969 net["name"]
970 )
971 if network_id == self._get_resource_name_from_resource_id(
972 net_ifaces["subnetwork"]
973 ):
974 net["vim_id"] = net_ifaces["name"]
975
976 self.logger.debug(
977 "new_vminstance Return: (name %s, created_items %s)",
978 vm_name,
979 created_items,
980 )
981 return vm_name, created_items
982
983 except Exception as e:
984 # Rollback vm creacion
985 if vm_id is not None:
986 try:
987 self.logger.debug("exception creating vm try to rollback")
988 self.delete_vminstance(vm_id, created_items)
989 except Exception as e2:
990 self.logger.error("new_vminstance rollback fail {}".format(e2))
991
992 else:
993 self.logger.debug(
994 "Exception creating new vminstance: %s", e, exc_info=True
995 )
996 self._format_vimconn_exception(e)
997
998 def _build_metadata(self, vm_name, cloud_config):
999 # initial metadata
1000 metadata = {}
1001 metadata["items"] = []
1002
1003 # if there is a cloud-init load it
1004 if cloud_config:
1005 self.logger.debug("cloud config: %s", cloud_config)
1006 _, userdata = self._create_user_data(cloud_config)
1007 metadata["items"].append({"key": "user-data", "value": userdata})
1008
1009 # either password of ssh-keys are required
1010 # we will always use ssh-keys, in case it is not available we will generate it
1011 self.logger.debug("metadata: %s", metadata)
1012
1013 return metadata
1014
1015 def get_vminstance(self, vm_id):
1016 """
1017 Obtaing the vm instance data from v_id
1018 """
1019 self.logger.debug("get_vminstance begin: vm_id %s", vm_id)
1020 self._reload_connection()
1021 response = {}
1022 try:
1023 response = (
1024 self.conn_compute.instances()
1025 .get(project=self.project, zone=self.zone, instance=vm_id)
1026 .execute()
1027 )
1028 # vm = response["source"]
1029 except Exception as e:
1030 self._format_vimconn_exception(e)
1031
1032 self.logger.debug("get_vminstance Return: response %s", response)
1033 return response
1034
1035 def delete_vminstance(self, vm_id, created_items=None, volumes_to_hold=None):
1036 """Deletes a vm instance from the vim."""
1037 self.logger.debug(
1038 "delete_vminstance begin: vm_id %s created_items %s",
1039 vm_id,
1040 created_items,
1041 )
1042 if self.reload_client:
1043 self._reload_connection()
1044
1045 created_items = created_items or {}
1046 try:
1047 vm = self.get_vminstance(vm_id)
1048
1049 operation = (
1050 self.conn_compute.instances()
1051 .delete(project=self.project, zone=self.zone, instance=vm_id)
1052 .execute()
1053 )
1054 self._wait_for_zone_operation(operation["name"])
1055
1056 # The associated subnets must be checked if they are marked to be deleted
1057 for netIface in vm["networkInterfaces"]:
1058 if (
1059 self._get_resource_name_from_resource_id(netIface["subnetwork"])
1060 in self.nets_to_be_deleted
1061 ):
1062 self._get_resource_name_from_resource_id(
1063 self.delete_network(netIface["subnetwork"])
1064 )
1065
1066 self.logger.debug("delete_vminstance end")
1067
1068 except Exception as e:
1069 # The VM can be deleted previously during network deletion
1070 if e.args[0]["status"] == "404":
1071 self.logger.debug("The VM doesn't exist or has been deleted")
1072 else:
1073 self._format_vimconn_exception(e)
1074
1075 def _get_resource_name_from_resource_id(self, resource_id):
1076 """
1077 Obtains resource_name from the google cloud complete identifier: resource_name will always be last item
1078 """
1079 self.logger.debug(
1080 "_get_resource_name_from_resource_id begin: resource_id %s",
1081 resource_id,
1082 )
1083 try:
1084 resource = str(resource_id.split("/")[-1])
1085
1086 self.logger.debug(
1087 "_get_resource_name_from_resource_id Return: resource %s",
1088 resource,
1089 )
1090 return resource
1091 except Exception as e:
1092 raise vimconn.VimConnException(
1093 "Unable to get resource name from resource_id '{}' Error: '{}'".format(
1094 resource_id, e
1095 )
1096 )
1097
1098 def refresh_nets_status(self, net_list):
1099 """Get the status of the networks
1100 Params: the list of network identifiers
1101 Returns a dictionary with:
1102 net_id: #VIM id of this network
1103 status: #Mandatory. Text with one of:
1104 # DELETED (not found at vim)
1105 # VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1106 # OTHER (Vim reported other status not understood)
1107 # ERROR (VIM indicates an ERROR status)
1108 # ACTIVE, INACTIVE, DOWN (admin down),
1109 # BUILD (on building process)
1110 #
1111 error_msg: #Text with VIM error message, if any. Or the VIM connection ERROR
1112 vim_info: #Text with plain information obtained from vim (yaml.safe_dump)
1113 """
1114 self.logger.debug("refresh_nets_status begin: net_list %s", net_list)
1115 out_nets = {}
1116 self._reload_connection()
1117
1118 for net_id in net_list:
1119 try:
1120 resName = self._get_resource_name_from_resource_id(net_id)
1121
1122 net = (
1123 self.conn_compute.subnetworks()
1124 .get(project=self.project, region=self.region, subnetwork=resName)
1125 .execute()
1126 )
1127 self.logger.debug("get subnetwork: %s", net)
1128
1129 out_nets[net_id] = {
1130 "status": "ACTIVE", # Google Cloud does not provide the status in subnetworks getting
1131 "vim_info": str(net),
1132 }
1133 except vimconn.VimConnNotFoundException as e:
1134 self.logger.error(
1135 "VimConnNotFoundException %s when searching subnet", e
1136 )
1137 out_nets[net_id] = {
1138 "status": "DELETED",
1139 "error_msg": str(e),
1140 }
1141 except Exception as e:
1142 self.logger.error(
1143 "Exception %s when searching subnet", e, exc_info=True
1144 )
1145 out_nets[net_id] = {
1146 "status": "VIM_ERROR",
1147 "error_msg": str(e),
1148 }
1149
1150 self.logger.debug("refresh_nets_status Return: out_nets %s", out_nets)
1151 return out_nets
1152
1153 def refresh_vms_status(self, vm_list):
1154 """Get the status of the virtual machines and their interfaces/ports
1155 Params: the list of VM identifiers
1156 Returns a dictionary with:
1157 vm_id: # VIM id of this Virtual Machine
1158 status: # Mandatory. Text with one of:
1159 # DELETED (not found at vim)
1160 # VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1161 # OTHER (Vim reported other status not understood)
1162 # ERROR (VIM indicates an ERROR status)
1163 # ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
1164 # BUILD (on building process), ERROR
1165 # ACTIVE:NoMgmtIP (Active but none of its interfaces has an IP address
1166 # (ACTIVE:NoMgmtIP is not returned for Azure)
1167 #
1168 error_msg: #Text with VIM error message, if any. Or the VIM connection ERROR
1169 vim_info: #Text with plain information obtained from vim (yaml.safe_dump)
1170 interfaces: list with interface info. Each item a dictionary with:
1171 vim_interface_id - The ID of the interface
1172 mac_address - The MAC address of the interface.
1173 ip_address - The IP address of the interface within the subnet.
1174 """
1175 self.logger.debug("refresh_vms_status begin: vm_list %s", vm_list)
1176 out_vms = {}
1177 self._reload_connection()
1178
1179 search_vm_list = vm_list or {}
1180
1181 for vm_id in search_vm_list:
1182 out_vm = {}
1183 try:
1184 res_name = self._get_resource_name_from_resource_id(vm_id)
1185
1186 vm = (
1187 self.conn_compute.instances()
1188 .get(project=self.project, zone=self.zone, instance=res_name)
1189 .execute()
1190 )
1191
1192 out_vm["vim_info"] = str(vm["name"])
1193 out_vm["status"] = self.provision_state2osm.get(vm["status"], "OTHER")
1194
1195 # In Google Cloud the there is no difference between provision or power status,
1196 # so if provision status method doesn't return a specific state (OTHER), the
1197 # power method is called
1198 if out_vm["status"] == "OTHER":
1199 out_vm["status"] = self.power_state2osm.get(vm["status"], "OTHER")
1200
1201 network_interfaces = vm["networkInterfaces"]
1202 out_vm["interfaces"] = self._get_vm_interfaces_status(
1203 vm_id, network_interfaces
1204 )
1205 except Exception as e:
1206 self.logger.error("Exception %s refreshing vm_status", e, exc_info=True)
1207 out_vm["status"] = "VIM_ERROR"
1208 out_vm["error_msg"] = str(e)
1209 out_vm["vim_info"] = None
1210
1211 out_vms[vm_id] = out_vm
1212
1213 self.logger.debug("refresh_vms_status Return: out_vms %s", out_vms)
1214 return out_vms
1215
1216 def _get_vm_interfaces_status(self, vm_id, interfaces):
1217 """
1218 Gets the interfaces detail for a vm
1219 :param interfaces: List of interfaces.
1220 :return: Dictionary with list of interfaces including, vim_interface_id, mac_address and ip_address
1221 """
1222 self.logger.debug(
1223 "_get_vm_interfaces_status begin: vm_id %s interfaces %s",
1224 vm_id,
1225 interfaces,
1226 )
1227 try:
1228 interface_list = []
1229 for network_interface in interfaces:
1230 interface_dict = {}
1231 interface_dict["vim_interface_id"] = network_interface["name"]
1232
1233 ips = []
1234 ips.append(network_interface["networkIP"])
1235 interface_dict["ip_address"] = ";".join(ips)
1236 interface_list.append(interface_dict)
1237
1238 self.logger.debug(
1239 "_get_vm_interfaces_status Return: interface_list %s",
1240 interface_list,
1241 )
1242 return interface_list
1243 except Exception as e:
1244 self.logger.error(
1245 "Exception %s obtaining interface data for vm: %s",
1246 e,
1247 vm_id,
1248 exc_info=True,
1249 )
1250 self._format_vimconn_exception(e)
1251
1252 def _create_firewall_rules(self, network):
1253 """
1254 Creates the necessary firewall rules to allow the traffic in the network
1255 (https://cloud.google.com/vpc/docs/firewalls)
1256 :param network.
1257 :return: a list with the names of the firewall rules
1258 """
1259 self.logger.debug("_create_firewall_rules begin: network %s", network)
1260 try:
1261 rules_list = []
1262
1263 # Adding firewall rule to allow http:
1264 self.logger.debug("creating firewall rule to allow http")
1265 firewall_rule_body = {
1266 "name": "fw-rule-http-" + network,
1267 "network": "global/networks/" + network,
1268 "allowed": [{"IPProtocol": "tcp", "ports": ["80"]}],
1269 }
1270 self.conn_compute.firewalls().insert(
1271 project=self.project, body=firewall_rule_body
1272 ).execute()
1273
1274 # Adding firewall rule to allow ssh:
1275 self.logger.debug("creating firewall rule to allow ssh")
1276 firewall_rule_body = {
1277 "name": "fw-rule-ssh-" + network,
1278 "network": "global/networks/" + network,
1279 "allowed": [{"IPProtocol": "tcp", "ports": ["22"]}],
1280 }
1281 self.conn_compute.firewalls().insert(
1282 project=self.project, body=firewall_rule_body
1283 ).execute()
1284
1285 # Adding firewall rule to allow ping:
1286 self.logger.debug("creating firewall rule to allow ping")
1287 firewall_rule_body = {
1288 "name": "fw-rule-icmp-" + network,
1289 "network": "global/networks/" + network,
1290 "allowed": [{"IPProtocol": "icmp"}],
1291 }
1292 self.conn_compute.firewalls().insert(
1293 project=self.project, body=firewall_rule_body
1294 ).execute()
1295
1296 # Adding firewall rule to allow internal:
1297 self.logger.debug("creating firewall rule to allow internal")
1298 firewall_rule_body = {
1299 "name": "fw-rule-internal-" + network,
1300 "network": "global/networks/" + network,
1301 "allowed": [
1302 {"IPProtocol": "tcp", "ports": ["0-65535"]},
1303 {"IPProtocol": "udp", "ports": ["0-65535"]},
1304 {"IPProtocol": "icmp"},
1305 ],
1306 }
1307 self.conn_compute.firewalls().insert(
1308 project=self.project, body=firewall_rule_body
1309 ).execute()
1310
1311 # Adding firewall rule to allow microk8s:
1312 self.logger.debug("creating firewall rule to allow microk8s")
1313 firewall_rule_body = {
1314 "name": "fw-rule-microk8s-" + network,
1315 "network": "global/networks/" + network,
1316 "allowed": [{"IPProtocol": "tcp", "ports": ["16443"]}],
1317 }
1318 self.conn_compute.firewalls().insert(
1319 project=self.project, body=firewall_rule_body
1320 ).execute()
1321
1322 # Adding firewall rule to allow rdp:
1323 self.logger.debug("creating firewall rule to allow rdp")
1324 firewall_rule_body = {
1325 "name": "fw-rule-rdp-" + network,
1326 "network": "global/networks/" + network,
1327 "allowed": [{"IPProtocol": "tcp", "ports": ["3389"]}],
1328 }
1329 self.conn_compute.firewalls().insert(
1330 project=self.project, body=firewall_rule_body
1331 ).execute()
1332
1333 # Adding firewall rule to allow osm:
1334 self.logger.debug("creating firewall rule to allow osm")
1335 firewall_rule_body = {
1336 "name": "fw-rule-osm-" + network,
1337 "network": "global/networks/" + network,
1338 "allowed": [{"IPProtocol": "tcp", "ports": ["9001", "9999"]}],
1339 }
1340 self.conn_compute.firewalls().insert(
1341 project=self.project, body=firewall_rule_body
1342 ).execute()
1343
1344 self.logger.debug(
1345 "_create_firewall_rules Return: list_rules %s", rules_list
1346 )
1347 return rules_list
1348 except Exception as e:
1349 self.logger.error(
1350 "Unable to create google cloud firewall rules for network '{}'".format(
1351 network
1352 )
1353 )
1354 self._format_vimconn_exception(e)
1355
1356 def _delete_firewall_rules(self, network):
1357 """
1358 Deletes the associated firewall rules to the network
1359 :param network.
1360 :return: a list with the names of the firewall rules
1361 """
1362 self.logger.debug("_delete_firewall_rules begin: network %s", network)
1363 try:
1364 rules_list = []
1365
1366 rules_list = (
1367 self.conn_compute.firewalls().list(project=self.project).execute()
1368 )
1369 for item in rules_list["items"]:
1370 if network == self._get_resource_name_from_resource_id(item["network"]):
1371 self.conn_compute.firewalls().delete(
1372 project=self.project, firewall=item["name"]
1373 ).execute()
1374
1375 self.logger.debug("_delete_firewall_rules Return: list_rules %s", 0)
1376 return rules_list
1377 except Exception as e:
1378 self.logger.error(
1379 "Unable to delete google cloud firewall rules for network '{}'".format(
1380 network
1381 )
1382 )
1383 self._format_vimconn_exception(e)
1384
1385 def migrate_instance(self, vm_id, compute_host=None):
1386 """
1387 Migrate a vdu
1388 param:
1389 vm_id: ID of an instance
1390 compute_host: Host to migrate the vdu to
1391 """
1392 # TODO: Add support for migration
1393 raise vimconn.VimConnNotImplemented("Not implemented")
1394
1395 def resize_instance(self, vm_id, flavor_id=None):
1396 """
1397 resize a vdu
1398 param:
1399 vm_id: ID of an instance
1400 flavor_id: flavor_id to resize the vdu to
1401 """
1402 # TODO: Add support for resize
1403 raise vimconn.VimConnNotImplemented("Not implemented")