97acfe762c53c8b71bfa1f22fad23ae8d97c231d
[osm/RO.git] / RO-VIM-gcp / osm_rovim_gcp / vimconn_gcp.py
1 # -*- coding: utf-8 -*-
2 ##
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
13 # under the License.
14 ##
15
16 import logging
17 from os import getenv
18 import random
19 from random import choice as random_choice
20 import time
21
22 from google.oauth2 import service_account
23 import googleapiclient.discovery
24 from osm_ro_plugin import vimconn
25
26 __author__ = "Sergio Gallardo Ruiz"
27 __date__ = "$11-aug-2021 08:30:00$"
28
29
30 if getenv("OSMRO_PDB_DEBUG"):
31 import sys
32
33 print(sys.path)
34 import pdb
35
36 pdb.set_trace()
37
38
39 class vimconnector(vimconn.VimConnector):
40
41 # Translate Google Cloud provisioning state to OSM provision state
42 # The first three ones are the transitional status once a user initiated action has been requested
43 # Once the operation is complete, it will transition into the states Succeeded or Failed
44 # https://cloud.google.com/compute/docs/instances/instance-life-cycle
45 provision_state2osm = {
46 "PROVISIONING": "BUILD",
47 "REPAIRING": "ERROR",
48 }
49
50 # Translate azure power state to OSM provision state
51 power_state2osm = {
52 "STAGING": "BUILD",
53 "RUNNING": "ACTIVE",
54 "STOPPING": "INACTIVE",
55 "SUSPENDING": "INACTIVE",
56 "SUSPENDED": "INACTIVE",
57 "TERMINATED": "INACTIVE",
58 }
59
60 # If a net or subnet is tried to be deleted and it has an associated resource, the net is marked "to be deleted"
61 # (incluid it's name in the following list). When the instance is deleted, its associated net will be deleted if
62 # they are present in that list
63 nets_to_be_deleted = []
64
65 def __init__(
66 self,
67 uuid,
68 name,
69 tenant_id,
70 tenant_name,
71 url,
72 url_admin=None,
73 user=None,
74 passwd=None,
75 log_level=None,
76 config={},
77 persistent_info={},
78 ):
79 """
80 Constructor of VIM. Raise an exception is some needed parameter is missing, but it must not do any connectivity
81 checking against the VIM
82 Using common constructor parameters.
83 In this case: config must include the following parameters:
84 subscription_id: assigned subscription identifier
85 region_name: current region for network
86 config may also include the following parameter:
87 flavors_pattern: pattern that will be used to select a range of vm sizes, for example
88 "^((?!Standard_B).)*$" will filter out Standard_B range that is cheap but is very overused
89 "^Standard_B" will select a serie B maybe for test environment
90 """
91 vimconn.VimConnector.__init__(
92 self,
93 uuid,
94 name,
95 tenant_id,
96 tenant_name,
97 url,
98 url_admin,
99 user,
100 passwd,
101 log_level,
102 config,
103 persistent_info,
104 )
105
106 # Variable that indicates if client must be reloaded or initialized
107 self.reload_client = False
108
109 # LOGGER
110
111 log_format_simple = (
112 "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
113 )
114 log_format_complete = "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(funcName)s(): %(message)s"
115 log_formatter_simple = logging.Formatter(
116 log_format_simple, datefmt="%Y-%m-%dT%H:%M:%S"
117 )
118 self.handler = logging.StreamHandler()
119 self.handler.setFormatter(log_formatter_simple)
120
121 self.logger = logging.getLogger("ro.vim.gcp")
122 self.logger.addHandler(self.handler)
123 if log_level:
124 self.logger.setLevel(getattr(logging, log_level))
125
126 if self.logger.getEffectiveLevel() == logging.DEBUG:
127 log_formatter = logging.Formatter(
128 log_format_complete, datefmt="%Y-%m-%dT%H:%M:%S"
129 )
130 self.handler.setFormatter(log_formatter)
131
132 self.logger.debug("Google Cloud connection init")
133
134 self.project = tenant_id or tenant_name
135
136 # REGION - Google Cloud considers regions and zones. A specific region can have more than one zone
137 # (for instance: region us-west1 with the zones us-west1-a, us-west1-b and us-west1-c)
138 # So the region name specified in the config will be considered as a specific zone for GC and
139 # the region will be calculated from that without the preffix.
140 if "region_name" in config:
141 self.zone = config.get("region_name")
142 self.region = self.zone.rsplit("-", 1)[0]
143 else:
144 raise vimconn.VimConnException(
145 "Google Cloud region_name is not specified at config"
146 )
147
148 # Credentials
149 self.logger.debug("Config: %s", config)
150 scopes = ["https://www.googleapis.com/auth/cloud-platform"]
151 self.credentials = None
152 if "credentials" in config:
153 self.logger.debug("Setting credentials")
154 # Settings Google Cloud credentials dict
155 credentials_body = config["credentials"]
156 # self.logger.debug("Credentials filtered: %s", credentials_body)
157 credentials = service_account.Credentials.from_service_account_info(
158 credentials_body
159 )
160 if "sa_file" in config:
161 credentials = service_account.Credentials.from_service_account_file(
162 config.get("sa_file"), scopes=scopes
163 )
164 self.logger.debug("Credentials: %s", credentials)
165 # Construct a Resource for interacting with an API.
166 self.credentials = credentials
167 try:
168 self.conn_compute = googleapiclient.discovery.build(
169 "compute", "v1", credentials=credentials
170 )
171 except Exception as e:
172 self._format_vimconn_exception(e)
173 else:
174 raise vimconn.VimConnException(
175 "It is not possible to init GCP with no credentials"
176 )
177
178 def _reload_connection(self):
179 """
180 Called before any operation, checks python Google Cloud clientsself.reload_client
181 """
182 if self.reload_client:
183 self.logger.debug("reloading google cloud client")
184
185 try:
186 # Set to client created
187 self.conn_compute = googleapiclient.discovery.build("compute", "v1")
188 except Exception as e:
189 self._format_vimconn_exception(e)
190
191 def _format_vimconn_exception(self, e):
192 """
193 Transforms a generic exception to a vimConnException
194 """
195 self.logger.error("Google Cloud plugin error: {}".format(e))
196 if isinstance(e, vimconn.VimConnException):
197 raise e
198 else:
199 # In case of generic error recreate client
200 self.reload_client = True
201 raise vimconn.VimConnException(type(e).__name__ + ": " + str(e))
202
203 def _wait_for_global_operation(self, operation):
204 """
205 Waits for the end of the specific operation
206 :operation: operation name
207 """
208
209 self.logger.debug("Waiting for operation %s", operation)
210
211 while True:
212 result = (
213 self.conn_compute.globalOperations()
214 .get(project=self.project, operation=operation)
215 .execute()
216 )
217
218 if result["status"] == "DONE":
219 if "error" in result:
220 raise vimconn.VimConnException(result["error"])
221 return result
222
223 time.sleep(1)
224
225 def _wait_for_zone_operation(self, operation):
226 """
227 Waits for the end of the specific operation
228 :operation: operation name
229 """
230
231 self.logger.debug("Waiting for operation %s", operation)
232
233 while True:
234 result = (
235 self.conn_compute.zoneOperations()
236 .get(project=self.project, operation=operation, zone=self.zone)
237 .execute()
238 )
239
240 if result["status"] == "DONE":
241 if "error" in result:
242 raise vimconn.VimConnException(result["error"])
243 return result
244
245 time.sleep(1)
246
247 def _wait_for_region_operation(self, operation):
248 """
249 Waits for the end of the specific operation
250 :operation: operation name
251 """
252
253 self.logger.debug("Waiting for operation %s", operation)
254
255 while True:
256 result = (
257 self.conn_compute.regionOperations()
258 .get(project=self.project, operation=operation, region=self.region)
259 .execute()
260 )
261
262 if result["status"] == "DONE":
263 if "error" in result:
264 raise vimconn.VimConnException(result["error"])
265 return result
266
267 time.sleep(1)
268
269 def new_network(
270 self,
271 net_name,
272 net_type,
273 ip_profile=None,
274 shared=False,
275 provider_network_profile=None,
276 ):
277 """
278 Adds a network to VIM
279 :param net_name: name of the network
280 :param net_type: not used for Google Cloud networks
281 :param ip_profile: not used for Google Cloud networks
282 :param shared: Not allowed for Google Cloud Connector
283 :param provider_network_profile: (optional)
284
285 contains {segmentation-id: vlan, provider-network: vim_netowrk}
286 :return: a tuple with the network identifier and created_items, or raises an exception on error
287 created_items can be None or a dictionary where this method can include key-values that will be passed to
288 the method delete_network. Can be used to store created segments, created l2gw connections, etc.
289 Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
290 as not present.
291 """
292
293 self.logger.debug(
294 "new_network begin: net_name %s net_type %s ip_profile %s shared %s provider_network_profile %s",
295 net_name,
296 net_type,
297 ip_profile,
298 shared,
299 provider_network_profile,
300 )
301 net_name = self._check_vm_name(net_name)
302 net_name = self._randomize_name(net_name)
303 self.logger.debug("create network name %s, ip_profile %s", net_name, ip_profile)
304
305 try:
306
307 self.logger.debug("creating network_name: {}".format(net_name))
308
309 network = "projects/{}/global/networks/default".format(self.project)
310 subnet_address = ""
311 if ip_profile is not None:
312 if "subnet_address" in ip_profile:
313 subnet_address = ip_profile["subnet_address"]
314 network_body = {
315 "name": str(net_name),
316 "description": net_name,
317 "network": network,
318 "ipCidrRange": subnet_address,
319 # The network is created in AUTO mode (one subnet per region is created)
320 # "autoCreateSubnetworks": True,
321 "autoCreateSubnetworks": False,
322 }
323
324 operation = (
325 self.conn_compute.networks()
326 .insert(project=self.project, body=network_body)
327 .execute()
328 )
329 self._wait_for_global_operation(operation["name"])
330 self.logger.debug("created network_name: {}".format(net_name))
331
332 # Adding firewall rules to allow the traffic in the network:
333 self._create_firewall_rules(net_name)
334
335 # create subnetwork, even if there is no profile
336
337 if not ip_profile:
338 ip_profile = {}
339
340 if not ip_profile.get("subnet_address"):
341 # Fake subnet is required
342 subnet_rand = random.randint(0, 255)
343 ip_profile["subnet_address"] = "192.168.{}.0/24".format(subnet_rand)
344
345 subnet_name = net_name + "-subnet"
346 subnet_id = self._new_subnet(
347 subnet_name, ip_profile, operation["targetLink"]
348 )
349
350 self.logger.debug("new_network Return: subnet_id: %s", subnet_id)
351 return subnet_id
352 except Exception as e:
353 self._format_vimconn_exception(e)
354
355 def _new_subnet(self, subnet_name, ip_profile, network):
356 """
357 Adds a tenant network to VIM. It creates a new subnet at existing base vnet
358 :param net_name: subnet name
359 :param ip_profile:
360 subnet-address: if it is not provided a subnet/24 in the default vnet is created,
361 otherwise it creates a subnet in the indicated address
362 :return: a tuple with the network identifier and created_items, or raises an exception on error
363 """
364 self.logger.debug(
365 "_new_subnet begin: subnet_name %s ip_profile %s network %s",
366 subnet_name,
367 ip_profile,
368 network,
369 )
370 self.logger.debug(
371 "create subnet name %s, ip_profile %s", subnet_name, ip_profile
372 )
373
374 try:
375
376 self.logger.debug("creating subnet_name: {}".format(subnet_name))
377
378 subnetwork_body = {
379 "name": str(subnet_name),
380 "description": subnet_name,
381 "network": network,
382 "ipCidrRange": ip_profile["subnet_address"],
383 }
384
385 operation = (
386 self.conn_compute.subnetworks()
387 .insert(
388 project=self.project,
389 region=self.region,
390 body=subnetwork_body,
391 )
392 .execute()
393 )
394 self._wait_for_region_operation(operation["name"])
395
396 self.logger.debug("created subnet_name: {}".format(subnet_name))
397
398 self.logger.debug(
399 "_new_subnet Return: (%s,%s)",
400 "regions/%s/subnetworks/%s" % (self.region, subnet_name),
401 None,
402 )
403 return "regions/%s/subnetworks/%s" % (self.region, subnet_name), None
404 except Exception as e:
405 self._format_vimconn_exception(e)
406
407 def get_network_list(self, filter_dict={}):
408 """Obtain tenant networks of VIM
409 Filter_dict can be:
410 name: network name
411 id: network id
412 shared: boolean, not implemented in GC
413 tenant_id: tenant, not used in GC, all networks same tenants
414 admin_state_up: boolean, not implemented in GC
415 status: 'ACTIVE', not implemented in GC #
416 Returns the network list of dictionaries
417 """
418 self.logger.debug("get_network_list begin: filter_dict %s", filter_dict)
419 self.logger.debug(
420 "Getting network (subnetwork) from VIM filter: {}".format(str(filter_dict))
421 )
422
423 try:
424
425 if self.reload_client:
426 self._reload_connection()
427
428 net_list = []
429
430 request = self.conn_compute.subnetworks().list(
431 project=self.project, region=self.region
432 )
433
434 while request is not None:
435 response = request.execute()
436 self.logger.debug("Network list: %s", response)
437 for net in response["items"]:
438 self.logger.debug("Network in list: {}".format(str(net["name"])))
439 if filter_dict is not None:
440 if "name" in filter_dict.keys():
441 if (
442 filter_dict["name"] == net["name"]
443 or filter_dict["name"] == net["selfLink"]
444 ):
445 self.logger.debug("Network found: %s", net["name"])
446 net_list.append(
447 {
448 "id": str(net["selfLink"]),
449 "name": str(net["name"]),
450 "network": str(net["network"]),
451 }
452 )
453 else:
454 net_list.append(
455 {
456 "id": str(net["selfLink"]),
457 "name": str(net["name"]),
458 "network": str(net["network"]),
459 }
460 )
461 request = self.conn_compute.subnetworks().list_next(
462 previous_request=request, previous_response=response
463 )
464
465 self.logger.debug("get_network_list Return: net_list %s", net_list)
466 return net_list
467
468 except Exception as e:
469 self.logger.error("Error in get_network_list()", exc_info=True)
470 raise vimconn.VimConnException(e)
471
472 def get_network(self, net_id):
473 self.logger.debug("get_network begin: net_id %s", net_id)
474 # res_name = self._get_resource_name_from_resource_id(net_id)
475 self._reload_connection()
476
477 self.logger.debug("Get network: %s", net_id)
478 filter_dict = {"name": net_id}
479 network_list = self.get_network_list(filter_dict)
480 self.logger.debug("Network list: %s", network_list)
481
482 if not network_list:
483 return []
484 else:
485 self.logger.debug("get_network Return: network_list[0] %s", network_list[0])
486 return network_list[0]
487
488 def delete_network(self, net_id, created_items=None):
489 """
490 Removes a tenant network from VIM and its associated elements
491 :param net_id: VIM identifier of the network, provided by method new_network
492 :param created_items: dictionary with extra items to be deleted. provided by method new_network
493 Returns the network identifier or raises an exception upon error or when network is not found
494 """
495
496 self.logger.debug(
497 "delete_network begin: net_id %s created_items %s",
498 net_id,
499 created_items,
500 )
501 self.logger.debug("Deleting network: {}".format(str(net_id)))
502
503 try:
504
505 net_name = self._get_resource_name_from_resource_id(net_id)
506
507 # Check associated VMs
508 self.conn_compute.instances().list(
509 project=self.project, zone=self.zone
510 ).execute()
511
512 net_id = self.delete_subnet(net_name, created_items)
513
514 self.logger.debug("delete_network Return: net_id %s", net_id)
515 return net_id
516
517 except Exception as e:
518 self.logger.error("Error in delete_network()", exc_info=True)
519 raise vimconn.VimConnException(e)
520
521 def delete_subnet(self, net_id, created_items=None):
522 """
523 Removes a tenant network from VIM and its associated elements
524 :param net_id: VIM identifier of the network, provided by method new_network
525 :param created_items: dictionary with extra items to be deleted. provided by method new_network
526 Returns the network identifier or raises an exception upon error or when network is not found
527 """
528
529 self.logger.debug(
530 "delete_subnet begin: net_id %s created_items %s",
531 net_id,
532 created_items,
533 )
534 self.logger.debug("Deleting subnetwork: {}".format(str(net_id)))
535
536 try:
537 # If the network has no more subnets, it will be deleted too
538 net_info = self.get_network(net_id)
539 # If the subnet is in use by another resource, the deletion will
540 # be retried N times before abort the operation
541 created_items = created_items or {}
542 created_items[net_id] = False
543
544 try:
545 operation = (
546 self.conn_compute.subnetworks()
547 .delete(
548 project=self.project,
549 region=self.region,
550 subnetwork=net_id,
551 )
552 .execute()
553 )
554 self._wait_for_region_operation(operation["name"])
555 if net_id in self.nets_to_be_deleted:
556 self.nets_to_be_deleted.remove(net_id)
557 except Exception as e:
558 if (
559 e.args[0]["status"] == "400"
560 ): # Resource in use, so the net is marked to be deleted
561 self.logger.debug("Subnet still in use")
562 self.nets_to_be_deleted.append(net_id)
563 else:
564 raise vimconn.VimConnException(e)
565
566 self.logger.debug("nets_to_be_deleted: %s", self.nets_to_be_deleted)
567
568 # If the network has no more subnets, it will be deleted too
569 # if "network" in net_info and net_id not in self.nets_to_be_deleted:
570 if "network" in net_info:
571 network_name = self._get_resource_name_from_resource_id(
572 net_info["network"]
573 )
574
575 try:
576 # Deletion of the associated firewall rules:
577 self._delete_firewall_rules(network_name)
578
579 operation = (
580 self.conn_compute.networks()
581 .delete(
582 project=self.project,
583 network=network_name,
584 )
585 .execute()
586 )
587 self._wait_for_global_operation(operation["name"])
588 except Exception as e:
589 self.logger.debug("error deleting associated network %s", e)
590
591 self.logger.debug("delete_subnet Return: net_id %s", net_id)
592 return net_id
593
594 except Exception as e:
595 self.logger.error("Error in delete_network()", exc_info=True)
596 raise vimconn.VimConnException(e)
597
598 def new_flavor(self, flavor_data):
599 """
600 It is not allowed to create new flavors (machine types) in Google Cloud, must always use an existing one
601 """
602 raise vimconn.VimConnNotImplemented(
603 "It is not possible to create new flavors in Google Cloud"
604 )
605
606 def new_tenant(self, tenant_name, tenant_description):
607 """
608 It is not allowed to create new tenants in Google Cloud
609 """
610 raise vimconn.VimConnNotImplemented(
611 "It is not possible to create a TENANT in Google Cloud"
612 )
613
614 def get_flavor(self, flavor_id):
615 """
616 Obtains the flavor_data from the flavor_id/machine type id
617 """
618 self.logger.debug("get_flavor begin: flavor_id %s", flavor_id)
619
620 try:
621 response = (
622 self.conn_compute.machineTypes()
623 .get(project=self.project, zone=self.zone, machineType=flavor_id)
624 .execute()
625 )
626 flavor_data = response
627 self.logger.debug("Machine type data: %s", flavor_data)
628
629 if flavor_data:
630 flavor = {
631 "id": flavor_data["id"],
632 "name": flavor_id,
633 "id_complete": flavor_data["selfLink"],
634 "ram": flavor_data["memoryMb"],
635 "vcpus": flavor_data["guestCpus"],
636 "disk": flavor_data["maximumPersistentDisksSizeGb"],
637 }
638
639 self.logger.debug("get_flavor Return: flavor %s", flavor)
640 return flavor
641 else:
642 raise vimconn.VimConnNotFoundException(
643 "flavor '{}' not found".format(flavor_id)
644 )
645 except Exception as e:
646 self._format_vimconn_exception(e)
647
648 # Google Cloud VM names can not have some special characters
649 def _check_vm_name(self, vm_name):
650 """
651 Checks vm name, in case the vm has not allowed characters they are removed, not error raised
652 Only lowercase and hyphens are allowed
653 """
654 chars_not_allowed_list = "~!@#$%^&*()=+_[]{}|;:<>/?."
655
656 # First: the VM name max length is 64 characters
657 vm_name_aux = vm_name[:62]
658
659 # Second: replace not allowed characters
660 for elem in chars_not_allowed_list:
661 # Check if string is in the main string
662 if elem in vm_name_aux:
663 # self.logger.debug("Dentro del IF")
664 # Replace the string
665 vm_name_aux = vm_name_aux.replace(elem, "-")
666
667 return vm_name_aux.lower()
668
669 def get_flavor_id_from_data(self, flavor_dict):
670 self.logger.debug("get_flavor_id_from_data begin: flavor_dict %s", flavor_dict)
671 filter_dict = flavor_dict or {}
672
673 try:
674 response = (
675 self.conn_compute.machineTypes()
676 .list(project=self.project, zone=self.zone)
677 .execute()
678 )
679 machine_types_list = response["items"]
680 # self.logger.debug("List of machine types: %s", machine_types_list)
681
682 cpus = filter_dict.get("vcpus") or 0
683 memMB = filter_dict.get("ram") or 0
684 # Workaround (it should be 0)
685 numberInterfaces = len(filter_dict.get("interfaces", [])) or 4
686
687 # Filter
688 filtered_machines = []
689 for machine_type in machine_types_list:
690 if (
691 machine_type["guestCpus"] >= cpus
692 and machine_type["memoryMb"] >= memMB
693 # In Google Cloud the number of virtual network interfaces scales with
694 # the number of virtual CPUs with a minimum of 2 and a maximum of 8:
695 # https://cloud.google.com/vpc/docs/create-use-multiple-interfaces#max-interfaces
696 and machine_type["guestCpus"] >= numberInterfaces
697 ):
698 filtered_machines.append(machine_type)
699
700 # self.logger.debug("Filtered machines: %s", filtered_machines)
701
702 # Sort
703 listedFilteredMachines = sorted(
704 filtered_machines,
705 key=lambda k: (
706 int(k["guestCpus"]),
707 float(k["memoryMb"]),
708 int(k["maximumPersistentDisksSizeGb"]),
709 k["name"],
710 ),
711 )
712 # self.logger.debug("Sorted filtered machines: %s", listedFilteredMachines)
713
714 if listedFilteredMachines:
715 self.logger.debug(
716 "get_flavor_id_from_data Return: listedFilteredMachines[0][name] %s",
717 listedFilteredMachines[0]["name"],
718 )
719 return listedFilteredMachines[0]["name"]
720
721 raise vimconn.VimConnNotFoundException(
722 "Cannot find any flavor matching '{}'".format(str(flavor_dict))
723 )
724
725 except Exception as e:
726 self._format_vimconn_exception(e)
727
728 def delete_flavor(self, flavor_id):
729 raise vimconn.VimConnNotImplemented(
730 "It is not possible to delete a flavor in Google Cloud"
731 )
732
733 def delete_tenant(self, tenant_id):
734 raise vimconn.VimConnNotImplemented(
735 "It is not possible to delete a TENANT in Google Cloud"
736 )
737
738 def new_image(self, image_dict):
739 """
740 This function comes from the early days when we though the image could be embedded in the package.
741 Unless OSM manages VM images E2E from NBI to RO, this function does not make sense to be implemented.
742 """
743 raise vimconn.VimConnNotImplemented("Not implemented")
744
745 def get_image_id_from_path(self, path):
746 """
747 This function comes from the early days when we though the image could be embedded in the package.
748 Unless OSM manages VM images E2E from NBI to RO, this function does not make sense to be implemented.
749 """
750 raise vimconn.VimConnNotImplemented("Not implemented")
751
752 def get_image_list(self, filter_dict={}):
753 """Obtain tenant images from VIM
754 Filter_dict can be:
755 name: image name with the format: image project:image family:image version
756 If some part of the name is provide ex: publisher:offer it will search all availables skus and version
757 for the provided publisher and offer
758 id: image uuid, currently not supported for azure
759 Returns the image list of dictionaries:
760 [{<the fields at Filter_dict plus some VIM specific>}, ...]
761 List can be empty
762 """
763 self.logger.debug("get_image_list begin: filter_dict %s", filter_dict)
764
765 try:
766 image_list = []
767 # Get image id from parameter image_id:
768 # <image Project>:image-family:<family> => Latest version of the family
769 # <image Project>:image:<image> => Specific image
770 # <image Project>:<image> => Specific image
771
772 image_info = filter_dict["name"].split(":")
773 image_project = image_info[0]
774 if len(image_info) == 2:
775 image_type = "image"
776 image_item = image_info[1]
777 if len(image_info) == 3:
778 image_type = image_info[1]
779 image_item = image_info[2]
780 else:
781 raise vimconn.VimConnNotFoundException("Wrong format for image")
782
783 image_response = {}
784 if image_type == "image-family":
785 image_response = (
786 self.conn_compute.images()
787 .getFromFamily(project=image_project, family=image_item)
788 .execute()
789 )
790 elif image_type == "image":
791 image_response = (
792 self.conn_compute.images()
793 .get(project=image_project, image=image_item)
794 .execute()
795 )
796 else:
797 raise vimconn.VimConnNotFoundException("Wrong format for image")
798 image_list.append(
799 {
800 "id": "projects/%s/global/images/%s"
801 % (image_project, image_response["name"]),
802 "name": ":".join(
803 [image_project, image_item, image_response["name"]]
804 ),
805 }
806 )
807
808 self.logger.debug("get_image_list Return: image_list %s", image_list)
809 return image_list
810
811 except Exception as e:
812 self._format_vimconn_exception(e)
813
814 def delete_image(self, image_id):
815 raise vimconn.VimConnNotImplemented("Not implemented")
816
817 def action_vminstance(self, vm_id, action_dict, created_items={}):
818 """Send and action over a VM instance from VIM
819 Returns the vm_id if the action was successfully sent to the VIM
820 """
821 raise vimconn.VimConnNotImplemented("Not necessary")
822
823 def _randomize_name(self, name):
824 """Adds a random string to allow requests with the same VM name
825 Returns the name with an additional random string (if the total size is bigger
826 than 62 the original name will be truncated)
827 """
828 random_name = name
829
830 while True:
831 try:
832 random_name = (
833 name[:49]
834 + "-"
835 + "".join(random_choice("0123456789abcdef") for _ in range(12))
836 )
837 self.conn_compute.instances().get(
838 project=self.project, zone=self.zone, instance=random_name
839 ).execute()
840 # If no exception is arisen, the random name exists for an instance,
841 # so a new random name must be generated
842
843 except Exception as e:
844 if e.args[0]["status"] == "404":
845 self.logger.debug("New random name: %s", random_name)
846 break
847 else:
848 self.logger.error(
849 "Exception generating random name (%s) for the instance", name
850 )
851 self._format_vimconn_exception(e)
852
853 return random_name
854
855 def new_vminstance(
856 self,
857 name,
858 description,
859 start,
860 image_id=None, # <image project>:(image|image-family):<image/family id>
861 flavor_id=None,
862 affinity_group_list=None,
863 net_list=None,
864 cloud_config=None,
865 disk_list=None,
866 availability_zone_index=None,
867 availability_zone_list=None,
868 ):
869 self.logger.debug(
870 "new_vminstance begin: name: %s, image_id: %s, flavor_id: %s, net_list: %s, cloud_config: %s, "
871 "disk_list: %s, availability_zone_index: %s, availability_zone_list: %s",
872 name,
873 image_id,
874 flavor_id,
875 net_list,
876 cloud_config,
877 disk_list,
878 availability_zone_index,
879 availability_zone_list,
880 )
881
882 if self.reload_client:
883 self._reload_connection()
884
885 # Validate input data is valid
886 # # First of all, the name must be adapted because Google Cloud only allows names consist of
887 # lowercase letters (a-z), numbers and hyphens (?:[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?)
888 vm_name = self._check_vm_name(name)
889 vm_name = self._randomize_name(vm_name)
890 vm_id = None
891
892 # At least one network must be provided
893 if not net_list:
894 raise vimconn.VimConnException(
895 "At least one net must be provided to create a new VM"
896 )
897
898 try:
899 created_items = {}
900 metadata = self._build_metadata(vm_name, cloud_config)
901
902 # Building network interfaces list
903 network_interfaces = []
904 for net in net_list:
905 net_iface = {}
906 if not net.get("net_id"):
907 if not net.get("name"):
908 continue
909 else:
910 net_iface[
911 "subnetwork"
912 ] = "regions/%s/subnetworks/" % self.region + net.get("name")
913 else:
914 net_iface["subnetwork"] = net.get("net_id")
915 # In order to get an external IP address, the key "accessConfigs" must be used
916 # in the interace. It has to be of type "ONE_TO_ONE_NAT" and name "External NAT"
917 if net.get("floating_ip", False) or (
918 net["use"] == "mgmt" and self.config.get("use_floating_ip")
919 ):
920 net_iface["accessConfigs"] = [
921 {"type": "ONE_TO_ONE_NAT", "name": "External NAT"}
922 ]
923
924 network_interfaces.append(net_iface)
925
926 self.logger.debug("Network interfaces: %s", network_interfaces)
927
928 self.logger.debug("Source image: %s", image_id)
929
930 vm_parameters = {
931 "name": vm_name,
932 "machineType": self.get_flavor(flavor_id)["id_complete"],
933 # Specify the boot disk and the image to use as a source.
934 "disks": [
935 {
936 "boot": True,
937 "autoDelete": True,
938 "initializeParams": {
939 "sourceImage": image_id,
940 },
941 }
942 ],
943 # Specify the network interfaces
944 "networkInterfaces": network_interfaces,
945 "metadata": metadata,
946 }
947
948 response = (
949 self.conn_compute.instances()
950 .insert(project=self.project, zone=self.zone, body=vm_parameters)
951 .execute()
952 )
953 self._wait_for_zone_operation(response["name"])
954
955 # The created instance info is obtained to get the name of the generated network interfaces (nic0, nic1...)
956 response = (
957 self.conn_compute.instances()
958 .get(project=self.project, zone=self.zone, instance=vm_name)
959 .execute()
960 )
961 self.logger.debug("instance get: %s", response)
962 vm_id = response["name"]
963
964 # The generated network interfaces in the instance are include in net_list:
965 for _, net in enumerate(net_list):
966 for net_ifaces in response["networkInterfaces"]:
967 network_id = ""
968 if "net_id" in net:
969 network_id = self._get_resource_name_from_resource_id(
970 net["net_id"]
971 )
972 else:
973 network_id = self._get_resource_name_from_resource_id(
974 net["name"]
975 )
976 if network_id == self._get_resource_name_from_resource_id(
977 net_ifaces["subnetwork"]
978 ):
979 net["vim_id"] = net_ifaces["name"]
980
981 self.logger.debug(
982 "new_vminstance Return: (name %s, created_items %s)",
983 vm_name,
984 created_items,
985 )
986 return vm_name, created_items
987
988 except Exception as e:
989 # Rollback vm creacion
990 if vm_id is not None:
991 try:
992 self.logger.debug("exception creating vm try to rollback")
993 self.delete_vminstance(vm_id, created_items)
994 except Exception as e2:
995 self.logger.error("new_vminstance rollback fail {}".format(e2))
996
997 else:
998 self.logger.debug(
999 "Exception creating new vminstance: %s", e, exc_info=True
1000 )
1001 self._format_vimconn_exception(e)
1002
1003 def _build_metadata(self, vm_name, cloud_config):
1004 # initial metadata
1005 metadata = {}
1006 metadata["items"] = []
1007
1008 # if there is a cloud-init load it
1009 if cloud_config:
1010 self.logger.debug("cloud config: %s", cloud_config)
1011 _, userdata = self._create_user_data(cloud_config)
1012 metadata["items"].append({"key": "user-data", "value": userdata})
1013
1014 # either password of ssh-keys are required
1015 # we will always use ssh-keys, in case it is not available we will generate it
1016 self.logger.debug("metadata: %s", metadata)
1017
1018 return metadata
1019
1020 def get_vminstance(self, vm_id):
1021 """
1022 Obtaing the vm instance data from v_id
1023 """
1024 self.logger.debug("get_vminstance begin: vm_id %s", vm_id)
1025 self._reload_connection()
1026 response = {}
1027 try:
1028 response = (
1029 self.conn_compute.instances()
1030 .get(project=self.project, zone=self.zone, instance=vm_id)
1031 .execute()
1032 )
1033 # vm = response["source"]
1034 except Exception as e:
1035 self._format_vimconn_exception(e)
1036
1037 self.logger.debug("get_vminstance Return: response %s", response)
1038 return response
1039
1040 def delete_vminstance(self, vm_id, created_items=None, volumes_to_hold=None):
1041 """Deletes a vm instance from the vim."""
1042 self.logger.debug(
1043 "delete_vminstance begin: vm_id %s created_items %s",
1044 vm_id,
1045 created_items,
1046 )
1047 if self.reload_client:
1048 self._reload_connection()
1049
1050 created_items = created_items or {}
1051 try:
1052 vm = self.get_vminstance(vm_id)
1053
1054 operation = (
1055 self.conn_compute.instances()
1056 .delete(project=self.project, zone=self.zone, instance=vm_id)
1057 .execute()
1058 )
1059 self._wait_for_zone_operation(operation["name"])
1060
1061 # The associated subnets must be checked if they are marked to be deleted
1062 for netIface in vm["networkInterfaces"]:
1063 if (
1064 self._get_resource_name_from_resource_id(netIface["subnetwork"])
1065 in self.nets_to_be_deleted
1066 ):
1067 self._get_resource_name_from_resource_id(
1068 self.delete_network(netIface["subnetwork"])
1069 )
1070
1071 self.logger.debug("delete_vminstance end")
1072
1073 except Exception as e:
1074 # The VM can be deleted previously during network deletion
1075 if e.args[0]["status"] == "404":
1076 self.logger.debug("The VM doesn't exist or has been deleted")
1077 else:
1078 self._format_vimconn_exception(e)
1079
1080 def _get_resource_name_from_resource_id(self, resource_id):
1081 """
1082 Obtains resource_name from the google cloud complete identifier: resource_name will always be last item
1083 """
1084 self.logger.debug(
1085 "_get_resource_name_from_resource_id begin: resource_id %s",
1086 resource_id,
1087 )
1088 try:
1089 resource = str(resource_id.split("/")[-1])
1090
1091 self.logger.debug(
1092 "_get_resource_name_from_resource_id Return: resource %s",
1093 resource,
1094 )
1095 return resource
1096 except Exception as e:
1097 raise vimconn.VimConnException(
1098 "Unable to get resource name from resource_id '{}' Error: '{}'".format(
1099 resource_id, e
1100 )
1101 )
1102
1103 def refresh_nets_status(self, net_list):
1104 """Get the status of the networks
1105 Params: the list of network identifiers
1106 Returns a dictionary with:
1107 net_id: #VIM id of this network
1108 status: #Mandatory. Text with one of:
1109 # DELETED (not found at vim)
1110 # VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1111 # OTHER (Vim reported other status not understood)
1112 # ERROR (VIM indicates an ERROR status)
1113 # ACTIVE, INACTIVE, DOWN (admin down),
1114 # BUILD (on building process)
1115 #
1116 error_msg: #Text with VIM error message, if any. Or the VIM connection ERROR
1117 vim_info: #Text with plain information obtained from vim (yaml.safe_dump)
1118 """
1119 self.logger.debug("refresh_nets_status begin: net_list %s", net_list)
1120 out_nets = {}
1121 self._reload_connection()
1122
1123 for net_id in net_list:
1124 try:
1125 resName = self._get_resource_name_from_resource_id(net_id)
1126
1127 net = (
1128 self.conn_compute.subnetworks()
1129 .get(project=self.project, region=self.region, subnetwork=resName)
1130 .execute()
1131 )
1132 self.logger.debug("get subnetwork: %s", net)
1133
1134 out_nets[net_id] = {
1135 "status": "ACTIVE", # Google Cloud does not provide the status in subnetworks getting
1136 "vim_info": str(net),
1137 }
1138 except vimconn.VimConnNotFoundException as e:
1139 self.logger.error(
1140 "VimConnNotFoundException %s when searching subnet", e
1141 )
1142 out_nets[net_id] = {
1143 "status": "DELETED",
1144 "error_msg": str(e),
1145 }
1146 except Exception as e:
1147 self.logger.error(
1148 "Exception %s when searching subnet", e, exc_info=True
1149 )
1150 out_nets[net_id] = {
1151 "status": "VIM_ERROR",
1152 "error_msg": str(e),
1153 }
1154
1155 self.logger.debug("refresh_nets_status Return: out_nets %s", out_nets)
1156 return out_nets
1157
1158 def refresh_vms_status(self, vm_list):
1159 """Get the status of the virtual machines and their interfaces/ports
1160 Params: the list of VM identifiers
1161 Returns a dictionary with:
1162 vm_id: # VIM id of this Virtual Machine
1163 status: # Mandatory. Text with one of:
1164 # DELETED (not found at vim)
1165 # VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1166 # OTHER (Vim reported other status not understood)
1167 # ERROR (VIM indicates an ERROR status)
1168 # ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
1169 # BUILD (on building process), ERROR
1170 # ACTIVE:NoMgmtIP (Active but none of its interfaces has an IP address
1171 # (ACTIVE:NoMgmtIP is not returned for Azure)
1172 #
1173 error_msg: #Text with VIM error message, if any. Or the VIM connection ERROR
1174 vim_info: #Text with plain information obtained from vim (yaml.safe_dump)
1175 interfaces: list with interface info. Each item a dictionary with:
1176 vim_interface_id - The ID of the interface
1177 mac_address - The MAC address of the interface.
1178 ip_address - The IP address of the interface within the subnet.
1179 """
1180 self.logger.debug("refresh_vms_status begin: vm_list %s", vm_list)
1181 out_vms = {}
1182 self._reload_connection()
1183
1184 search_vm_list = vm_list or {}
1185
1186 for vm_id in search_vm_list:
1187 out_vm = {}
1188 try:
1189 res_name = self._get_resource_name_from_resource_id(vm_id)
1190
1191 vm = (
1192 self.conn_compute.instances()
1193 .get(project=self.project, zone=self.zone, instance=res_name)
1194 .execute()
1195 )
1196
1197 out_vm["vim_info"] = str(vm["name"])
1198 out_vm["status"] = self.provision_state2osm.get(vm["status"], "OTHER")
1199
1200 # In Google Cloud the there is no difference between provision or power status,
1201 # so if provision status method doesn't return a specific state (OTHER), the
1202 # power method is called
1203 if out_vm["status"] == "OTHER":
1204 out_vm["status"] = self.power_state2osm.get(vm["status"], "OTHER")
1205
1206 network_interfaces = vm["networkInterfaces"]
1207 out_vm["interfaces"] = self._get_vm_interfaces_status(
1208 vm_id, network_interfaces
1209 )
1210 except Exception as e:
1211 self.logger.error("Exception %s refreshing vm_status", e, exc_info=True)
1212 out_vm["status"] = "VIM_ERROR"
1213 out_vm["error_msg"] = str(e)
1214 out_vm["vim_info"] = None
1215
1216 out_vms[vm_id] = out_vm
1217
1218 self.logger.debug("refresh_vms_status Return: out_vms %s", out_vms)
1219 return out_vms
1220
1221 def _get_vm_interfaces_status(self, vm_id, interfaces):
1222 """
1223 Gets the interfaces detail for a vm
1224 :param interfaces: List of interfaces.
1225 :return: Dictionary with list of interfaces including, vim_interface_id, mac_address and ip_address
1226 """
1227 self.logger.debug(
1228 "_get_vm_interfaces_status begin: vm_id %s interfaces %s",
1229 vm_id,
1230 interfaces,
1231 )
1232 try:
1233 interface_list = []
1234 for network_interface in interfaces:
1235 interface_dict = {}
1236 interface_dict["vim_interface_id"] = network_interface["name"]
1237
1238 ips = []
1239 ips.append(network_interface["networkIP"])
1240 interface_dict["ip_address"] = ";".join(ips)
1241 interface_list.append(interface_dict)
1242
1243 self.logger.debug(
1244 "_get_vm_interfaces_status Return: interface_list %s",
1245 interface_list,
1246 )
1247 return interface_list
1248 except Exception as e:
1249 self.logger.error(
1250 "Exception %s obtaining interface data for vm: %s",
1251 e,
1252 vm_id,
1253 exc_info=True,
1254 )
1255 self._format_vimconn_exception(e)
1256
1257 def _create_firewall_rules(self, network):
1258 """
1259 Creates the necessary firewall rules to allow the traffic in the network
1260 (https://cloud.google.com/vpc/docs/firewalls)
1261 :param network.
1262 :return: a list with the names of the firewall rules
1263 """
1264 self.logger.debug("_create_firewall_rules begin: network %s", network)
1265 try:
1266 rules_list = []
1267
1268 # Adding firewall rule to allow http:
1269 self.logger.debug("creating firewall rule to allow http")
1270 firewall_rule_body = {
1271 "name": "fw-rule-http-" + network,
1272 "network": "global/networks/" + network,
1273 "allowed": [{"IPProtocol": "tcp", "ports": ["80"]}],
1274 }
1275 self.conn_compute.firewalls().insert(
1276 project=self.project, body=firewall_rule_body
1277 ).execute()
1278
1279 # Adding firewall rule to allow ssh:
1280 self.logger.debug("creating firewall rule to allow ssh")
1281 firewall_rule_body = {
1282 "name": "fw-rule-ssh-" + network,
1283 "network": "global/networks/" + network,
1284 "allowed": [{"IPProtocol": "tcp", "ports": ["22"]}],
1285 }
1286 self.conn_compute.firewalls().insert(
1287 project=self.project, body=firewall_rule_body
1288 ).execute()
1289
1290 # Adding firewall rule to allow ping:
1291 self.logger.debug("creating firewall rule to allow ping")
1292 firewall_rule_body = {
1293 "name": "fw-rule-icmp-" + network,
1294 "network": "global/networks/" + network,
1295 "allowed": [{"IPProtocol": "icmp"}],
1296 }
1297 self.conn_compute.firewalls().insert(
1298 project=self.project, body=firewall_rule_body
1299 ).execute()
1300
1301 # Adding firewall rule to allow internal:
1302 self.logger.debug("creating firewall rule to allow internal")
1303 firewall_rule_body = {
1304 "name": "fw-rule-internal-" + network,
1305 "network": "global/networks/" + network,
1306 "allowed": [
1307 {"IPProtocol": "tcp", "ports": ["0-65535"]},
1308 {"IPProtocol": "udp", "ports": ["0-65535"]},
1309 {"IPProtocol": "icmp"},
1310 ],
1311 }
1312 self.conn_compute.firewalls().insert(
1313 project=self.project, body=firewall_rule_body
1314 ).execute()
1315
1316 # Adding firewall rule to allow microk8s:
1317 self.logger.debug("creating firewall rule to allow microk8s")
1318 firewall_rule_body = {
1319 "name": "fw-rule-microk8s-" + network,
1320 "network": "global/networks/" + network,
1321 "allowed": [{"IPProtocol": "tcp", "ports": ["16443"]}],
1322 }
1323 self.conn_compute.firewalls().insert(
1324 project=self.project, body=firewall_rule_body
1325 ).execute()
1326
1327 # Adding firewall rule to allow rdp:
1328 self.logger.debug("creating firewall rule to allow rdp")
1329 firewall_rule_body = {
1330 "name": "fw-rule-rdp-" + network,
1331 "network": "global/networks/" + network,
1332 "allowed": [{"IPProtocol": "tcp", "ports": ["3389"]}],
1333 }
1334 self.conn_compute.firewalls().insert(
1335 project=self.project, body=firewall_rule_body
1336 ).execute()
1337
1338 # Adding firewall rule to allow osm:
1339 self.logger.debug("creating firewall rule to allow osm")
1340 firewall_rule_body = {
1341 "name": "fw-rule-osm-" + network,
1342 "network": "global/networks/" + network,
1343 "allowed": [{"IPProtocol": "tcp", "ports": ["9001", "9999"]}],
1344 }
1345 self.conn_compute.firewalls().insert(
1346 project=self.project, body=firewall_rule_body
1347 ).execute()
1348
1349 self.logger.debug(
1350 "_create_firewall_rules Return: list_rules %s", rules_list
1351 )
1352 return rules_list
1353 except Exception as e:
1354 self.logger.error(
1355 "Unable to create google cloud firewall rules for network '{}'".format(
1356 network
1357 )
1358 )
1359 self._format_vimconn_exception(e)
1360
1361 def _delete_firewall_rules(self, network):
1362 """
1363 Deletes the associated firewall rules to the network
1364 :param network.
1365 :return: a list with the names of the firewall rules
1366 """
1367 self.logger.debug("_delete_firewall_rules begin: network %s", network)
1368 try:
1369 rules_list = []
1370
1371 rules_list = (
1372 self.conn_compute.firewalls().list(project=self.project).execute()
1373 )
1374 for item in rules_list["items"]:
1375 if network == self._get_resource_name_from_resource_id(item["network"]):
1376 self.conn_compute.firewalls().delete(
1377 project=self.project, firewall=item["name"]
1378 ).execute()
1379
1380 self.logger.debug("_delete_firewall_rules Return: list_rules %s", 0)
1381 return rules_list
1382 except Exception as e:
1383 self.logger.error(
1384 "Unable to delete google cloud firewall rules for network '{}'".format(
1385 network
1386 )
1387 )
1388 self._format_vimconn_exception(e)
1389
1390 def migrate_instance(self, vm_id, compute_host=None):
1391 """
1392 Migrate a vdu
1393 param:
1394 vm_id: ID of an instance
1395 compute_host: Host to migrate the vdu to
1396 """
1397 # TODO: Add support for migration
1398 raise vimconn.VimConnNotImplemented("Not implemented")
1399
1400 def resize_instance(self, vm_id, flavor_id=None):
1401 """
1402 resize a vdu
1403 param:
1404 vm_id: ID of an instance
1405 flavor_id: flavor_id to resize the vdu to
1406 """
1407 # TODO: Add support for resize
1408 raise vimconn.VimConnNotImplemented("Not implemented")