RO-VIM-aws/osm_rovim_aws/vimconn_aws.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2017 xFlow Research Pvt. Ltd
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: saboor.ahmad@xflowresearch.com
  22 ##
  23
  24 """
  25 AWS-connector implements all the methods to interact with AWS using the BOTO client
  26 """
  27 import logging
  28 import random
  29 import time
  30 import traceback
  31
  32 import boto
  33 import boto.ec2
  34 from boto.exception import BotoServerError, EC2ResponseError
  35 import boto.vpc
  36 from ipconflict import check_conflicts
  37 import netaddr
  38 from osm_ro_plugin import vimconn
  39 import yaml
  40
  41 __author__ = "Saboor Ahmad"
  42 __date__ = "10-Apr-2017"
  43
  44
  45 class vimconnector(vimconn.VimConnector):
  46     def __init__(
  47         self,
  48         uuid,
  49         name,
  50         tenant_id,
  51         tenant_name,
  52         url,
  53         url_admin=None,
  54         user=None,
  55         passwd=None,
  56         log_level=None,
  57         config={},
  58         persistent_info={},
  59     ):
  60         """Params:
  61         uuid - id asigned to this VIM
  62         name - name assigned to this VIM, can be used for logging
  63         tenant_id - ID to be used for tenant
  64         tenant_name - name of tenant to be used VIM tenant to be used
  65         url_admin - optional, url used for administrative tasks
  66         user - credentials of the VIM user
  67         passwd - credentials of the VIM user
  68         log_level - if must use a different log_level than the general one
  69         config - dictionary with misc VIM information
  70             region_name - name of region to deploy the instances
  71             vpc_cidr_block - default CIDR block for VPC
  72             security_groups - default security group to specify this instance
  73         persistent_info - dict where the class can store information that will be available among class
  74             destroy/creation cycles. This info is unique per VIM/credential. At first call it will contain an
  75             empty dict. Useful to store login/tokens information for speed up communication
  76         """
  77         vimconn.VimConnector.__init__(
  78             self,
  79             uuid,
  80             name,
  81             tenant_id,
  82             tenant_name,
  83             url,
  84             url_admin,
  85             user,
  86             passwd,
  87             log_level,
  88             config,
  89             persistent_info,
  90         )
  91
  92         self.persistent_info = persistent_info
  93         self.a_creds = {}
  94
  95         if user:
  96             self.a_creds["aws_access_key_id"] = user
  97         else:
  98             raise vimconn.VimConnAuthException("Username is not specified")
  99
 100         if passwd:
 101             self.a_creds["aws_secret_access_key"] = passwd
 102         else:
 103             raise vimconn.VimConnAuthException("Password is not specified")
 104
 105         if "region_name" in config:
 106             self.region = config.get("region_name")
 107         else:
 108             raise vimconn.VimConnException("AWS region_name is not specified at config")
 109
 110         self.vpc_data = {}
 111         self.subnet_data = {}
 112         self.conn = None
 113         self.conn_vpc = None
 114         self.account_id = None
 115         self.network_delete_on_termination = []
 116         self.server_timeout = 180
 117
 118         self.vpc_id = self.get_tenant_list()[0]["id"]
 119         # we take VPC CIDR block if specified, otherwise we use the default CIDR
 120         # block suggested by AWS while creating instance
 121         self.vpc_cidr_block = "10.0.0.0/24"
 122
 123         if tenant_name:
 124             self.vpc_id = tenant_name
 125
 126         if "vpc_cidr_block" in config:
 127             self.vpc_cidr_block = config["vpc_cidr_block"]
 128
 129         self.security_groups = None
 130         if "security_groups" in config:
 131             self.security_groups = config["security_groups"]
 132
 133         self.key_pair = None
 134         if "key_pair" in config:
 135             self.key_pair = config["key_pair"]
 136
 137         self.flavor_info = None
 138         if "flavor_info" in config:
 139             flavor_data = config.get("flavor_info")
 140             if isinstance(flavor_data, str):
 141                 try:
 142                     if flavor_data[0] == "@":  # read from a file
 143                         with open(flavor_data[1:], "r") as stream:
 144                             self.flavor_info = yaml.safe_load(stream)
 145                     else:
 146                         self.flavor_info = yaml.safe_load(flavor_data)
 147                 except yaml.YAMLError as e:
 148                     self.flavor_info = None
 149
 150                     raise vimconn.VimConnException(
 151                         "Bad format at file '{}': {}".format(flavor_data[1:], e)
 152                     )
 153                 except IOError as e:
 154                     raise vimconn.VimConnException(
 155                         "Error reading file '{}': {}".format(flavor_data[1:], e)
 156                     )
 157             elif isinstance(flavor_data, dict):
 158                 self.flavor_info = flavor_data
 159
 160         self.logger = logging.getLogger("ro.vim.aws")
 161
 162         if log_level:
 163             self.logger.setLevel(getattr(logging, log_level))
 164
 165     def __setitem__(self, index, value):
 166         """Params:
 167         index - name of value of set
 168         value - value to set
 169         """
 170         if index == "user":
 171             self.a_creds["aws_access_key_id"] = value
 172         elif index == "passwd":
 173             self.a_creds["aws_secret_access_key"] = value
 174         elif index == "region":
 175             self.region = value
 176         else:
 177             vimconn.VimConnector.__setitem__(self, index, value)
 178
 179     def _reload_connection(self):
 180         """Returns: sets boto.EC2 and boto.VPC connection to work with AWS services"""
 181         try:
 182             self.conn = boto.ec2.connect_to_region(
 183                 self.region,
 184                 aws_access_key_id=self.a_creds["aws_access_key_id"],
 185                 aws_secret_access_key=self.a_creds["aws_secret_access_key"],
 186             )
 187             self.conn_vpc = boto.vpc.connect_to_region(
 188                 self.region,
 189                 aws_access_key_id=self.a_creds["aws_access_key_id"],
 190                 aws_secret_access_key=self.a_creds["aws_secret_access_key"],
 191             )
 192             # client = boto3.client("sts", aws_access_key_id=self.a_creds['aws_access_key_id'],
 193             # aws_secret_access_key=self.a_creds['aws_secret_access_key'])
 194             # self.account_id = client.get_caller_identity()["Account"]
 195         except Exception as e:
 196             self.format_vimconn_exception(e)
 197
 198     def format_vimconn_exception(self, e):
 199         """Params: an Exception object
 200         Returns: Raises the exception 'e' passed in mehtod parameters
 201         """
 202         self.conn = None
 203         self.conn_vpc = None
 204
 205         raise vimconn.VimConnConnectionException(type(e).__name__ + ": " + str(e))
 206
 207     def get_tenant_list(self, filter_dict={}):
 208         """Obtain tenants of VIM
 209         filter_dict dictionary that can contain the following keys:
 210             name: filter by tenant name
 211             id: filter by tenant uuid/id
 212             <other VIM specific>
 213         Returns the tenant list of dictionaries, and empty list if no tenant match all the filers:
 214             [{'name':'<name>, 'id':'<id>, ...}, ...]
 215         """
 216         try:
 217             self._reload_connection()
 218             vpc_ids = []
 219
 220             if filter_dict != {}:
 221                 if "id" in filter_dict:
 222                     vpc_ids.append(filter_dict["id"])
 223
 224             tenants = self.conn_vpc.get_all_vpcs(vpc_ids, None)
 225             tenant_list = []
 226
 227             for tenant in tenants:
 228                 tenant_list.append(
 229                     {
 230                         "id": str(tenant.id),
 231                         "name": str(tenant.id),
 232                         "status": str(tenant.state),
 233                         "cidr_block": str(tenant.cidr_block),
 234                     }
 235                 )
 236
 237             return tenant_list
 238         except Exception as e:
 239             self.format_vimconn_exception(e)
 240
 241     def new_tenant(self, tenant_name, tenant_description):
 242         """Adds a new tenant to VIM with this name and description, this is done using admin_url if provided
 243         "tenant_name": string max lenght 64
 244         "tenant_description": string max length 256
 245         returns the tenant identifier or raise exception
 246         """
 247         self.logger.debug("Adding a new VPC")
 248
 249         try:
 250             self._reload_connection()
 251             vpc = self.conn_vpc.create_vpc(self.vpc_cidr_block)
 252             self.conn_vpc.modify_vpc_attribute(vpc.id, enable_dns_support=True)
 253             self.conn_vpc.modify_vpc_attribute(vpc.id, enable_dns_hostnames=True)
 254
 255             gateway = self.conn_vpc.create_internet_gateway()
 256             self.conn_vpc.attach_internet_gateway(gateway.id, vpc.id)
 257             route_table = self.conn_vpc.create_route_table(vpc.id)
 258             self.conn_vpc.create_route(route_table.id, "0.0.0.0/0", gateway.id)
 259
 260             self.vpc_data[vpc.id] = {
 261                 "gateway": gateway.id,
 262                 "route_table": route_table.id,
 263                 "subnets": self.subnet_sizes(self.vpc_cidr_block),
 264             }
 265
 266             return vpc.id
 267         except Exception as e:
 268             self.format_vimconn_exception(e)
 269
 270     def delete_tenant(self, tenant_id):
 271         """Delete a tenant from VIM
 272         tenant_id: returned VIM tenant_id on "new_tenant"
 273         Returns None on success. Raises and exception of failure. If tenant is not found raises vimconnNotFoundException
 274         """
 275         self.logger.debug("Deleting specified VPC")
 276
 277         try:
 278             self._reload_connection()
 279             vpc = self.vpc_data.get(tenant_id)
 280
 281             if "gateway" in vpc and "route_table" in vpc:
 282                 gateway_id, route_table_id = vpc["gateway"], vpc["route_table"]
 283                 self.conn_vpc.detach_internet_gateway(gateway_id, tenant_id)
 284                 self.conn_vpc.delete_vpc(tenant_id)
 285                 self.conn_vpc.delete_route(route_table_id, "0.0.0.0/0")
 286             else:
 287                 self.conn_vpc.delete_vpc(tenant_id)
 288         except Exception as e:
 289             self.format_vimconn_exception(e)
 290
 291     def subnet_sizes(self, cidr):
 292         """Calculates possible subnets given CIDR value of VPC"""
 293         netmasks = (
 294             "255.255.0.0",
 295             "255.255.128.0",
 296             "255.255.192.0",
 297             "255.255.224.0",
 298             "255.255.240.0",
 299             "255.255.248.0",
 300         )
 301
 302         ip = netaddr.IPNetwork(cidr)
 303         mask = ip.netmask
 304         pub_split = ()
 305
 306         for netmask in netmasks:
 307             if str(mask) == netmask:
 308                 pub_split = list(ip.subnet(24))
 309                 break
 310
 311         subnets = pub_split if pub_split else (list(ip.subnet(28)))
 312
 313         return map(str, subnets)
 314
 315     def new_network(
 316         self,
 317         net_name,
 318         net_type,
 319         ip_profile=None,
 320         shared=False,
 321         provider_network_profile=None,
 322     ):
 323         """Adds a tenant network to VIM
 324         Params:
 325             'net_name': name of the network
 326             'net_type': one of:
 327                 'bridge': overlay isolated network
 328                 'data':   underlay E-LAN network for Passthrough and SRIOV interfaces
 329                 'ptp':    underlay E-LINE network for Passthrough and SRIOV interfaces.
 330             'ip_profile': is a dict containing the IP parameters of the network (Currently only IPv4 is implemented)
 331                 'ip-version': can be one of ["IPv4","IPv6"]
 332                 'subnet-address': ip_prefix_schema, that is X.X.X.X/Y
 333                 'gateway-address': (Optional) ip_schema, that is X.X.X.X
 334                 'dns-address': (Optional) ip_schema,
 335                 'dhcp': (Optional) dict containing
 336                     'enabled': {"type": "boolean"},
 337                     'start-address': ip_schema, first IP to grant
 338                     'count': number of IPs to grant.
 339             'shared': if this network can be seen/use by other tenants/organization
 340         Returns a tuple with the network identifier and created_items, or raises an exception on error
 341             created_items can be None or a dictionary where this method can include key-values that will be passed to
 342             the method delete_network. Can be used to store created segments, created l2gw connections, etc.
 343             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
 344             as not present.
 345         """
 346         self.logger.debug("Adding a subnet to VPC")
 347
 348         try:
 349             created_items = {}
 350             self._reload_connection()
 351             subnet = None
 352             vpc_id = self.vpc_id
 353             if self.conn_vpc.get_all_subnets():
 354                 existing_subnet = self.conn_vpc.get_all_subnets()[0]
 355                 if not self.availability_zone:
 356                     self.availability_zone = str(existing_subnet.availability_zone)
 357
 358             if self.vpc_data.get(vpc_id, None):
 359                 cidr_block = list(
 360                     set(self.vpc_data[vpc_id]["subnets"])
 361                     - set(
 362                         self.get_network_details(
 363                             {"tenant_id": vpc_id}, detail="cidr_block"
 364                         )
 365                     )
 366                 )
 367             else:
 368                 vpc = self.get_tenant_list({"id": vpc_id})[0]
 369                 subnet_list = self.subnet_sizes(vpc["cidr_block"])
 370                 cidr_block = list(
 371                     set(subnet_list)
 372                     - set(
 373                         self.get_network_details(
 374                             {"tenant_id": vpc["id"]}, detail="cidr_block"
 375                         )
 376                     )
 377                 )
 378
 379             try:
 380                 selected_cidr_block = random.choice(cidr_block)
 381                 retry = 15
 382                 while retry > 0:
 383                     all_subnets = [
 384                         subnet.cidr_block for subnet in self.conn_vpc.get_all_subnets()
 385                     ]
 386                     all_subnets.append(selected_cidr_block)
 387                     conflict = check_conflicts(all_subnets)
 388                     if not conflict:
 389                         subnet = self.conn_vpc.create_subnet(
 390                             vpc_id, selected_cidr_block, self.availability_zone
 391                         )
 392                         break
 393                     retry -= 1
 394                     selected_cidr_block = random.choice(cidr_block)
 395                 else:
 396                     raise vimconn.VimConnException(
 397                         "Failed to find a proper CIDR which does not overlap"
 398                         "with existing subnets",
 399                         http_code=vimconn.HTTP_Request_Timeout,
 400                     )
 401
 402             except (EC2ResponseError, BotoServerError) as error:
 403                 self.format_vimconn_exception(error)
 404
 405             created_items["net:" + str(subnet.id)] = True
 406
 407             return subnet.id, created_items
 408         except Exception as e:
 409             self.format_vimconn_exception(e)
 410
 411     def get_network_details(self, filters, detail):
 412         """Get specified details related to a subnet"""
 413         detail_list = []
 414         subnet_list = self.get_network_list(filters)
 415
 416         for net in subnet_list:
 417             detail_list.append(net[detail])
 418
 419         return detail_list
 420
 421     def get_network_list(self, filter_dict={}):
 422         """Obtain tenant networks of VIM
 423         Params:
 424             'filter_dict' (optional) contains entries to return only networks that matches ALL entries:
 425                 name: string  => returns only networks with this name
 426                 id:   string  => returns networks with this VIM id, this imply returns one network at most
 427                 shared: boolean >= returns only networks that are (or are not) shared
 428                 tenant_id: sting => returns only networks that belong to this tenant/project
 429                 ,#(not used yet) admin_state_up: boolean => returns only networks that are (or are not) in admin
 430                     state active
 431                 #(not used yet) status: 'ACTIVE','ERROR',... => filter networks that are on this status
 432         Returns the network list of dictionaries. each dictionary contains:
 433             'id': (mandatory) VIM network id
 434             'name': (mandatory) VIM network name
 435             'status': (mandatory) can be 'ACTIVE', 'INACTIVE', 'DOWN', 'BUILD', 'ERROR', 'VIM_ERROR', 'OTHER'
 436             'error_msg': (optional) text that explains the ERROR status
 437             other VIM specific fields: (optional) whenever possible using the same naming of filter_dict param
 438         List can be empty if no network map the filter_dict. Raise an exception only upon VIM connectivity,
 439             authorization, or some other unspecific error
 440         """
 441         self.logger.debug("Getting all subnets from VIM")
 442
 443         try:
 444             self._reload_connection()
 445             tfilters = {}
 446
 447             if filter_dict != {}:
 448                 if "tenant_id" in filter_dict:
 449                     tfilters["vpcId"] = filter_dict.get("tenant_id")
 450
 451             subnets = self.conn_vpc.get_all_subnets(
 452                 subnet_ids=filter_dict.get("SubnetId", None), filters=tfilters
 453             )
 454
 455             net_list = []
 456
 457             for net in subnets:
 458                 if net.id == filter_dict.get("name"):
 459                     self.availability_zone = str(net.availability_zone)
 460                     net_list.append(
 461                         {
 462                             "id": str(net.id),
 463                             "name": str(net.id),
 464                             "status": str(net.state),
 465                             "vpc_id": str(net.vpc_id),
 466                             "cidr_block": str(net.cidr_block),
 467                             "type": "bridge",
 468                         }
 469                     )
 470
 471             return net_list
 472         except Exception as e:
 473             self.format_vimconn_exception(e)
 474
 475     def get_network(self, net_id):
 476         """Obtain network details from the 'net_id' VIM network
 477         Return a dict that contains:
 478             'id': (mandatory) VIM network id, that is, net_id
 479             'name': (mandatory) VIM network name
 480             'status': (mandatory) can be 'ACTIVE', 'INACTIVE', 'DOWN', 'BUILD', 'ERROR', 'VIM_ERROR', 'OTHER'
 481             'error_msg': (optional) text that explains the ERROR status
 482             other VIM specific fields: (optional) whenever possible using the same naming of filter_dict param
 483         Raises an exception upon error or when network is not found
 484         """
 485         self.logger.debug("Getting Subnet from VIM")
 486
 487         try:
 488             self._reload_connection()
 489             subnet = self.conn_vpc.get_all_subnets(net_id)[0]
 490             return {
 491                 "id": str(subnet.id),
 492                 "name": str(subnet.id),
 493                 "status": str(subnet.state),
 494                 "vpc_id": str(subnet.vpc_id),
 495                 "cidr_block": str(subnet.cidr_block),
 496                 "availability_zone": str(subnet.availability_zone),
 497             }
 498         except Exception as e:
 499             self.format_vimconn_exception(e)
 500
 501     def delete_network(self, net_id, created_items=None):
 502         """
 503         Removes a tenant network from VIM and its associated elements
 504         :param net_id: VIM identifier of the network, provided by method new_network
 505         :param created_items: dictionary with extra items to be deleted. provided by method new_network
 506         Returns the network identifier or raises an exception upon error or when network is not found
 507         """
 508         self.logger.debug("Deleting subnet from VIM")
 509
 510         try:
 511             self._reload_connection()
 512             self.logger.debug("DELETING NET_ID: " + str(net_id))
 513             self.conn_vpc.delete_subnet(net_id)
 514
 515             return net_id
 516
 517         except Exception as e:
 518             if isinstance(e, EC2ResponseError):
 519                 self.network_delete_on_termination.append(net_id)
 520                 self.logger.warning(
 521                     f"{net_id} could not be deleted, deletion will retry after dependencies resolved"
 522                 )
 523             else:
 524                 self.format_vimconn_exception(e)
 525
 526     def refresh_nets_status(self, net_list):
 527         """Get the status of the networks
 528         Params:
 529             'net_list': a list with the VIM network id to be get the status
 530         Returns a dictionary with:
 531             'net_id':         #VIM id of this network
 532                 status:     #Mandatory. Text with one of:
 533                     #  DELETED (not found at vim)
 534                     #  VIM_ERROR (Cannot connect to VIM, authentication problems, VIM response error, ...)
 535                     #  OTHER (Vim reported other status not understood)
 536                     #  ERROR (VIM indicates an ERROR status)
 537                     #  ACTIVE, INACTIVE, DOWN (admin down),
 538                     #  BUILD (on building process)
 539                 error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 540                 vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 541             'net_id2': ...
 542         """
 543         self._reload_connection()
 544
 545         try:
 546             dict_entry = {}
 547
 548             for net_id in net_list:
 549                 subnet_dict = {}
 550                 subnet = None
 551
 552                 try:
 553                     subnet = self.conn_vpc.get_all_subnets(net_id)[0]
 554
 555                     if subnet.state == "pending":
 556                         subnet_dict["status"] = "BUILD"
 557                     elif subnet.state == "available":
 558                         subnet_dict["status"] = "ACTIVE"
 559                     else:
 560                         subnet_dict["status"] = "ERROR"
 561                     subnet_dict["error_msg"] = ""
 562                 except Exception:
 563                     subnet_dict["status"] = "DELETED"
 564                     subnet_dict["error_msg"] = "Network not found"
 565                 finally:
 566                     subnet_dictionary = vars(subnet)
 567                     cleared_subnet_dict = {
 568                         key: subnet_dictionary[key]
 569                         for key in subnet_dictionary
 570                         if not isinstance(subnet_dictionary[key], object)
 571                     }
 572                     subnet_dict["vim_info"] = cleared_subnet_dict
 573                 dict_entry[net_id] = subnet_dict
 574
 575             return dict_entry
 576         except Exception as e:
 577             self.format_vimconn_exception(e)
 578
 579     def get_flavor(self, flavor_id):
 580         """Obtain flavor details from the VIM
 581         Returns the flavor dict details {'id':<>, 'name':<>, other vim specific }
 582         Raises an exception upon error or if not found
 583         """
 584         self.logger.debug("Getting instance type")
 585
 586         try:
 587             if flavor_id in self.flavor_info:
 588                 return self.flavor_info[flavor_id]
 589             else:
 590                 raise vimconn.VimConnNotFoundException(
 591                     "Cannot find flavor with this flavor ID/Name"
 592                 )
 593         except Exception as e:
 594             self.format_vimconn_exception(e)
 595
 596     def new_image(self, image_dict):
 597         """Adds a tenant image to VIM
 598         Params: image_dict
 599             name (string) - The name of the AMI. Valid only for EBS-based images.
 600             description (string) - The description of the AMI.
 601             image_location (string) - Full path to your AMI manifest in Amazon S3 storage. Only used for S3-based AMI’s.
 602             architecture (string) - The architecture of the AMI. Valid choices are: * i386 * x86_64
 603             kernel_id (string) -  The ID of the kernel with which to launch the instances
 604             root_device_name (string) - The root device name (e.g. /dev/sdh)
 605             block_device_map (boto.ec2.blockdevicemapping.BlockDeviceMapping) - A BlockDeviceMapping data structure
 606                 describing the EBS volumes associated with the Image.
 607             virtualization_type (string) - The virutalization_type of the image. Valid choices are: * paravirtual * hvm
 608             sriov_net_support (string) - Advanced networking support. Valid choices are: * simple
 609             snapshot_id (string) - A snapshot ID for the snapshot to be used as root device for the image. Mutually
 610                 exclusive with block_device_map, requires root_device_name
 611             delete_root_volume_on_termination (bool) - Whether to delete the root volume of the image after instance
 612                 termination. Only applies when creating image from snapshot_id. Defaults to False. Note that leaving
 613                     volumes behind after instance termination is not free
 614         Returns: image_id - image ID of the newly created image
 615         """
 616         try:
 617             self._reload_connection()
 618             image_location = image_dict.get("image_location", None)
 619
 620             if image_location:
 621                 image_location = str(self.account_id) + str(image_location)
 622
 623             image_id = self.conn.register_image(
 624                 image_dict.get("name", None),
 625                 image_dict.get("description", None),
 626                 image_location,
 627                 image_dict.get("architecture", None),
 628                 image_dict.get("kernel_id", None),
 629                 image_dict.get("root_device_name", None),
 630                 image_dict.get("block_device_map", None),
 631                 image_dict.get("virtualization_type", None),
 632                 image_dict.get("sriov_net_support", None),
 633                 image_dict.get("snapshot_id", None),
 634                 image_dict.get("delete_root_volume_on_termination", None),
 635             )
 636
 637             return image_id
 638         except Exception as e:
 639             self.format_vimconn_exception(e)
 640
 641     def delete_image(self, image_id):
 642         """Deletes a tenant image from VIM
 643         Returns the image_id if image is deleted or raises an exception on error"""
 644
 645         try:
 646             self._reload_connection()
 647             self.conn.deregister_image(image_id)
 648
 649             return image_id
 650         except Exception as e:
 651             self.format_vimconn_exception(e)
 652
 653     def get_image_id_from_path(self, path):
 654         """
 655         Params: path - location of the image
 656         Returns: image_id - ID of the matching image
 657         """
 658         self._reload_connection()
 659         try:
 660             filters = {}
 661
 662             if path:
 663                 tokens = path.split("/")
 664                 filters["owner_id"] = tokens[0]
 665                 filters["name"] = "/".join(tokens[1:])
 666
 667             image = self.conn.get_all_images(filters=filters)[0]
 668
 669             return image.id
 670         except Exception as e:
 671             self.format_vimconn_exception(e)
 672
 673     def get_image_list(self, filter_dict={}):
 674         """Obtain tenant images from VIM
 675         Filter_dict can be:
 676             name: image name
 677             id: image uuid
 678             checksum: image checksum
 679             location: image path
 680         Returns the image list of dictionaries:
 681             [{<the fields at Filter_dict plus some VIM specific>}, ...]
 682             List can be empty
 683         """
 684         self.logger.debug("Getting image list from VIM")
 685
 686         try:
 687             self._reload_connection()
 688             image_id = None
 689             filters = {}
 690
 691             if "id" in filter_dict:
 692                 image_id = filter_dict["id"]
 693
 694             if "name" in filter_dict:
 695                 filters["name"] = filter_dict["name"]
 696
 697             if "location" in filter_dict:
 698                 filters["location"] = filter_dict["location"]
 699
 700             # filters['image_type'] = 'machine'
 701             # filter_dict['owner_id'] = self.account_id
 702             images = self.conn.get_all_images(image_id, filters=filters)
 703             image_list = []
 704
 705             for image in images:
 706                 image_list.append(
 707                     {
 708                         "id": str(image.id),
 709                         "name": str(image.name),
 710                         "status": str(image.state),
 711                         "owner": str(image.owner_id),
 712                         "location": str(image.location),
 713                         "is_public": str(image.is_public),
 714                         "architecture": str(image.architecture),
 715                         "platform": str(image.platform),
 716                     }
 717                 )
 718
 719             return image_list
 720         except Exception as e:
 721             self.format_vimconn_exception(e)
 722
 723     def new_vminstance(
 724         self,
 725         name,
 726         description,
 727         start,
 728         image_id,
 729         flavor_id,
 730         affinity_group_list,
 731         net_list,
 732         cloud_config=None,
 733         disk_list=None,
 734         availability_zone_index=None,
 735         availability_zone_list=None,
 736     ):
 737         """Create a new VM/instance in AWS
 738         Params: name
 739                 decription
 740                 start: (boolean) indicates if VM must start or created in pause mode.
 741                 image_id - image ID in AWS
 742                 flavor_id - instance type ID in AWS
 743                 net_list
 744                     name
 745                     net_id - subnet_id from AWS
 746                     vpci - (optional) virtual vPCI address to assign at the VM. Can be ignored depending on VIM
 747                          capabilities
 748                     model: (optional and only have sense for type==virtual) interface model: virtio, e1000, ...
 749                     mac_address: (optional) mac address to assign to this interface
 750                     type: (mandatory) can be one of:
 751                         virtual, in this case always connected to a network of type 'net_type=bridge'
 752                         'PCI-PASSTHROUGH' or 'PF' (passthrough): depending on VIM capabilities it can be connected to a
 753                              data/ptp network ot it
 754                            can created unconnected
 755                         'SR-IOV' or 'VF' (SRIOV with VLAN tag): same as PF for network connectivity.
 756                         VFnotShared - (SRIOV without VLAN tag) same as PF for network connectivity. VF where no other
 757                             VFs are allocated on the same physical NIC
 758                     bw': (optional) only for PF/VF/VFnotShared. Minimal Bandwidth required for the interface in GBPS
 759                     port_security': (optional) If False it must avoid any traffic filtering at this interface.
 760                          If missing or True, it must apply the default VIM behaviour
 761                     vim_id': must be filled/added by this method with the VIM identifier generated by the VIM for this
 762                          interface. 'net_list' is modified
 763                     elastic_ip - True/False to define if an elastic_ip is required
 764                 cloud_config': (optional) dictionary with:
 765                     key-pairs': (optional) list of strings with the public key to be inserted to the default user
 766                     users': (optional) list of users to be inserted, each item is a dict with:
 767                         name': (mandatory) user name,
 768                         key-pairs': (optional) list of strings with the public key to be inserted to the user
 769                     user-data': (optional) string is a text script to be passed directly to cloud-init
 770                     config-files': (optional). List of files to be transferred. Each item is a dict with:
 771                         dest': (mandatory) string with the destination absolute path
 772                         encoding': (optional, by default text). Can be one of:
 773                             b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64'
 774                         content' (mandatory): string with the content of the file
 775                         permissions': (optional) string with file permissions, typically octal notation '0644'
 776                         owner: (optional) file owner, string with the format 'owner:group'
 777                     boot-data-drive: boolean to indicate if user-data must be passed using a boot drive (hard disk)
 778                     security-groups:
 779                         subnet_id
 780                         security_group_id
 781                 disk_list': (optional) list with additional disks to the VM. Each item is a dict with:
 782                     image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted
 783                     size': (mandatory) string with the size of the disk in GB
 784         Returns a tuple with the instance identifier and created_items or raises an exception on error
 785             created_items can be None or a dictionary where this method can include key-values that will be passed to
 786             the method delete_vminstance and action_vminstance. Can be used to store created ports, volumes, etc.
 787             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
 788             as not present.
 789         """
 790         self.logger.debug("Creating a new VM instance")
 791
 792         try:
 793             created_items = {}
 794             self._reload_connection()
 795             reservation = None
 796             _, userdata = self._create_user_data(cloud_config)
 797
 798             if not net_list:
 799                 reservation = self.conn.run_instances(
 800                     image_id,
 801                     key_name=self.key_pair,
 802                     instance_type=flavor_id,
 803                     security_groups=self.security_groups,
 804                     user_data=userdata,
 805                 )
 806
 807             else:
 808                 for index, subnet in enumerate(net_list):
 809                     net_intr = self.conn_vpc.create_network_interface(
 810                         subnet_id=subnet.get("net_id"),
 811                         groups=None,
 812                     )
 813
 814                     interface = boto.ec2.networkinterface.NetworkInterfaceSpecification(
 815                         network_interface_id=net_intr.id,
 816                         device_index=index,
 817                     )
 818
 819                     interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(
 820                         interface
 821                     )
 822
 823                     if subnet.get("elastic_ip"):
 824                         eip = self.conn.allocate_address()
 825                         self.conn.associate_address(
 826                             allocation_id=eip.allocation_id,
 827                             network_interface_id=net_intr.id,
 828                         )
 829
 830                     if index == 0:
 831                         try:
 832                             reservation = self.conn.run_instances(
 833                                 image_id,
 834                                 key_name=self.key_pair,
 835                                 instance_type=flavor_id,
 836                                 security_groups=self.security_groups,
 837                                 network_interfaces=interfaces,
 838                                 user_data=userdata,
 839                             )
 840                         except Exception as instance_create_error:
 841                             self.logger.debug(traceback.format_exc())
 842                             self.format_vimconn_exception(instance_create_error)
 843
 844                     if index > 0:
 845                         try:
 846                             if reservation:
 847                                 instance_id = self.wait_for_instance_id(reservation)
 848                                 if instance_id and self.wait_for_vm(
 849                                     instance_id, "running"
 850                                 ):
 851                                     self.conn.attach_network_interface(
 852                                         network_interface_id=net_intr.id,
 853                                         instance_id=instance_id,
 854                                         device_index=index,
 855                                     )
 856                         except Exception as attach_network_error:
 857                             self.logger.debug(traceback.format_exc())
 858                             self.format_vimconn_exception(attach_network_error)
 859
 860                     if instance_id := self.wait_for_instance_id(reservation):
 861                         time.sleep(30)
 862                         instance_status = self.refresh_vms_status(instance_id)
 863                         refreshed_instance_status = instance_status.get(instance_id)
 864                         instance_interfaces = refreshed_instance_status.get(
 865                             "interfaces"
 866                         )
 867                         for idx, interface in enumerate(instance_interfaces):
 868                             if idx == index:
 869                                 net_list[index]["vim_id"] = instance_interfaces[
 870                                     idx
 871                                 ].get("vim_interface_id")
 872
 873             instance_id = self.wait_for_instance_id(reservation)
 874             created_items["vm_id:" + str(instance_id)] = True
 875
 876             return instance_id, created_items
 877         except Exception as e:
 878             self.logger.debug(traceback.format_exc())
 879             self.format_vimconn_exception(e)
 880
 881     def get_vminstance(self, vm_id):
 882         """Returns the VM instance information from VIM"""
 883         try:
 884             self._reload_connection()
 885             reservation = self.conn.get_all_instances(vm_id)
 886
 887             return reservation[0].instances[0].__dict__
 888         except Exception as e:
 889             self.format_vimconn_exception(e)
 890
 891     def delete_vminstance(self, vm_id, created_items=None, volumes_to_hold=None):
 892         """Removes a VM instance from VIM
 893         Returns the instance identifier"""
 894         try:
 895             self._reload_connection()
 896             self.logger.debug("DELETING VM_ID: " + str(vm_id))
 897             reservation = self.conn.get_all_instances(vm_id)[0]
 898             if hasattr(reservation, "instances"):
 899                 instance = reservation.instances[0]
 900
 901                 self.conn.terminate_instances(vm_id)
 902                 if self.wait_for_vm(vm_id, "terminated"):
 903                     for interface in instance.interfaces:
 904                         self.conn_vpc.delete_network_interface(
 905                             network_interface_id=interface.id,
 906                         )
 907                 if self.network_delete_on_termination:
 908                     for net in self.network_delete_on_termination:
 909                         try:
 910                             self.conn_vpc.delete_subnet(net)
 911                         except Exception as net_delete_error:
 912                             if isinstance(net_delete_error, EC2ResponseError):
 913                                 self.logger.warning(f"Deleting network {net}: failed")
 914                             else:
 915                                 self.format_vimconn_exception(net_delete_error)
 916
 917                 return vm_id
 918         except Exception as e:
 919             self.format_vimconn_exception(e)
 920
 921     def wait_for_instance_id(self, reservation):
 922         if not reservation:
 923             return False
 924
 925         self._reload_connection()
 926         elapsed_time = 0
 927         while elapsed_time < 30:
 928             if reservation.instances:
 929                 instance_id = reservation.instances[0].id
 930                 return instance_id
 931             time.sleep(5)
 932             elapsed_time += 5
 933         else:
 934             raise vimconn.VimConnException(
 935                 "Failed to get instance_id for reservation",
 936                 http_code=vimconn.HTTP_Request_Timeout,
 937             )
 938
 939     def wait_for_vm(self, vm_id, status):
 940         """wait until vm is in the desired status and return True.
 941         If the timeout is reached generate an exception"""
 942
 943         self._reload_connection()
 944
 945         elapsed_time = 0
 946         while elapsed_time < self.server_timeout:
 947             if self.conn.get_all_instances(vm_id):
 948                 reservation = self.conn.get_all_instances(vm_id)[0]
 949                 if hasattr(reservation, "instances"):
 950                     instance = reservation.instances[0]
 951                     if instance.state == status:
 952                         return True
 953             time.sleep(5)
 954             elapsed_time += 5
 955
 956         # if we exceeded the timeout
 957         else:
 958             raise vimconn.VimConnException(
 959                 "Timeout waiting for instance " + vm_id + " to get " + status,
 960                 http_code=vimconn.HTTP_Request_Timeout,
 961             )
 962
 963     def refresh_vms_status(self, vm_list):
 964         """Get the status of the virtual machines and their interfaces/ports
 965         Params: the list of VM identifiers
 966         Returns a dictionary with:
 967             vm_id:          #VIM id of this Virtual Machine
 968                 status:     #Mandatory. Text with one of:
 969                             #  DELETED (not found at vim)
 970                             #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 971                             #  OTHER (Vim reported other status not understood)
 972                             #  ERROR (VIM indicates an ERROR status)
 973                             #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
 974                             #  BUILD (on building process), ERROR
 975                             #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
 976                             #
 977                 error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 978                 vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 979                 interfaces: list with interface info. Each item a dictionary with:
 980                     vim_interface_id -  The ID of the ENI.
 981                     vim_net_id - The ID of the VPC subnet.
 982                     mac_address - The MAC address of the interface.
 983                     ip_address - The IP address of the interface within the subnet.
 984         """
 985         self.logger.debug("Getting VM instance information from VIM")
 986
 987         try:
 988             self._reload_connection()
 989             elapsed_time = 0
 990             while elapsed_time < self.server_timeout:
 991                 reservation = self.conn.get_all_instances(vm_list)[0]
 992                 if reservation:
 993                     break
 994                 time.sleep(5)
 995                 elapsed_time += 5
 996
 997             # if we exceeded the timeout
 998             else:
 999                 raise vimconn.VimConnException(
1000                     vm_list + "could not be gathered, refresh vm status failed",
1001                     http_code=vimconn.HTTP_Request_Timeout,
1002                 )
1003
1004             instances = {}
1005             instance_dict = {}
1006
1007             for instance in reservation.instances:
1008                 if hasattr(instance, "id"):
1009                     try:
1010                         if instance.state in ("pending"):
1011                             instance_dict["status"] = "BUILD"
1012                         elif instance.state in ("available", "running", "up"):
1013                             instance_dict["status"] = "ACTIVE"
1014                         else:
1015                             instance_dict["status"] = "ERROR"
1016
1017                         instance_dict["error_msg"] = ""
1018                         instance_dict["interfaces"] = []
1019
1020                         for interface in instance.interfaces:
1021                             interface_dict = {
1022                                 "vim_interface_id": interface.id,
1023                                 "vim_net_id": interface.subnet_id,
1024                                 "mac_address": interface.mac_address,
1025                             }
1026
1027                             if (
1028                                 hasattr(interface, "publicIp")
1029                                 and interface.publicIp is not None
1030                             ):
1031                                 interface_dict["ip_address"] = (
1032                                     interface.publicIp
1033                                     + ";"
1034                                     + interface.private_ip_address
1035                                 )
1036                             else:
1037                                 interface_dict["ip_address"] = (
1038                                     interface.private_ip_address
1039                                 )
1040
1041                             instance_dict["interfaces"].append(interface_dict)
1042                     except Exception as e:
1043                         self.logger.error(
1044                             "Exception getting vm status: %s", str(e), exc_info=True
1045                         )
1046                         instance_dict["status"] = "DELETED"
1047                         instance_dict["error_msg"] = str(e)
1048                     finally:
1049                         instance_dictionary = vars(instance)
1050                         cleared_instance_dict = {
1051                             key: instance_dictionary[key]
1052                             for key in instance_dictionary
1053                             if not (isinstance(instance_dictionary[key], object))
1054                         }
1055                         instance_dict["vim_info"] = cleared_instance_dict
1056
1057                     instances[instance.id] = instance_dict
1058
1059             return instances
1060         except Exception as e:
1061             self.logger.error("Exception getting vm status: %s", str(e), exc_info=True)
1062             self.format_vimconn_exception(e)
1063
1064     def action_vminstance(self, vm_id, action_dict, created_items={}):
1065         """Send and action over a VM instance from VIM
1066         Returns the vm_id if the action was successfully sent to the VIM"""
1067
1068         self.logger.debug("Action over VM '%s': %s", vm_id, str(action_dict))
1069         try:
1070             self._reload_connection()
1071             if "start" in action_dict:
1072                 self.conn.start_instances(vm_id)
1073             elif "stop" in action_dict or "stop" in action_dict:
1074                 self.conn.stop_instances(vm_id)
1075             elif "terminate" in action_dict:
1076                 self.conn.terminate_instances(vm_id)
1077             elif "reboot" in action_dict:
1078                 self.conn.reboot_instances(vm_id)
1079
1080             return None
1081         except Exception as e:
1082             self.format_vimconn_exception(e)
1083
1084     def migrate_instance(self, vm_id, compute_host=None):
1085         """
1086         Migrate a vdu
1087         param:
1088             vm_id: ID of an instance
1089             compute_host: Host to migrate the vdu to
1090         """
1091         # TODO: Add support for migration
1092         raise vimconn.VimConnNotImplemented("Not implemented")
1093
1094     def resize_instance(self, vm_id, flavor_id=None):
1095         """
1096         resize a vdu
1097         param:
1098             vm_id: ID of an instance
1099             flavor_id: flavor to resize the vdu
1100         """
1101         # TODO: Add support for resize
1102         raise vimconn.VimConnNotImplemented("Not implemented")