rwcal/test/ec2.py

   1
   2 #
   3 #   Copyright 2016 RIFT.IO Inc
   4 #
   5 #   Licensed under the Apache License, Version 2.0 (the "License");
   6 #   you may not use this file except in compliance with the License.
   7 #   You may obtain a copy of the License at
   8 #
   9 #       http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 #   Unless required by applicable law or agreed to in writing, software
  12 #   distributed under the License is distributed on an "AS IS" BASIS,
  13 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 #   See the License for the specific language governing permissions and
  15 #   limitations under the License.
  16 #
  17
  18 import glob
  19 import itertools
  20 import os
  21
  22 import boto
  23 import boto.vpc
  24
  25 # TODO:  Pull the lastest of owned instances.
  26 __default_instance_ami__ = 'ami-e421bc8c'
  27
  28 # TODO:  Make VPC's per user?
  29 __default_subnet__ = 'subnet-4b484363'
  30 __default_security_group__ = 'sg-d9da90bc'
  31
  32 __default_instance_type__ = 'm1.medium'
  33 __default_vpc__ = 'vpc-e7ed4482'
  34
  35 class RWEC2(object):
  36     def __init__(self,  subnet=None, ami=None):
  37         self._subnet = subnet if subnet is not None else __default_subnet__
  38         self._ami = ami if ami is not None else __default_instance_ami__
  39
  40         self._conn = boto.connect_ec2()
  41
  42     @staticmethod
  43     def cloud_init_current_user():
  44         """
  45         Return user_data configuration suitable for cloud-init that will create a user
  46         with sudo and ssh key access on the remote instance.
  47
  48         ssh keys are found with the glob ~/.ssh/*pub*
  49         """
  50         user_data = "users:\n"
  51         user_data += " - name: %s\n" % (os.getlogin(),)
  52         user_data += "   groups: [wheel, adm, systemd-journal]\n"
  53         user_data += "   sudo: [\"ALL=(ALL) NOPASSWD:ALL\"]\n"
  54         user_data += "   shell: /bin/bash\n"
  55         user_data += "   ssh_authorized_keys:\n"
  56         for pub_key in glob.glob('%s/.ssh/*pub*' % (os.environ['HOME'],)):
  57             with open(pub_key) as fp:
  58                 user_data += "    -  %s" % (fp.read(),)
  59
  60         return user_data
  61
  62
  63     @staticmethod
  64     def cloud_init_yum_repos():
  65         """
  66         Return a string of user_data commands that can be used to update the yum
  67         repos to point to the correct location.  They should be added by the caller
  68         within a 'runcmd:' block.
  69         """
  70         ret = " - sed -i -e 's,www\.,,' -e 's,riftio\.com/mirrors,riftio.com:8881,' /etc/yum.repos.d/*.repo\n"
  71         return ret
  72
  73     def instances(self, cluster_component, cluster_instance):
  74         """
  75         List of instances owned by the given cluster instance
  76
  77         @param cluster_component  - parent cluster of each instance
  78         @param cluster_instance   - instance id of the owning cluster
  79         @param n_instances        - number of requested instances
  80
  81         @return                   - list of boto.ec2.instance.Instances provisioned
  82         """
  83         ret = []
  84         reservations = self._conn.get_all_instances()
  85         for instance in [instance for reservation in reservations for instance in reservation.instances]:
  86             tags = instance.tags
  87             if (tags.get('parent_component') == cluster_component
  88                     and tags.get('parent_instance') == cluster_instance):
  89                 ret.append(instance)
  90
  91         return ret
  92
  93     def provision_master(self, cluster_component, cluster_instance):
  94         """
  95         Provision a master instance in EC2.  The master instance is a special instance with the
  96         following features:
  97             - Public IP
  98             - /home shared over NFS
  99
 100         @param cluster_component  - parent cluster of each instance
 101         @param cluster_instance   - instance id of the owning cluster
 102
 103         @return                   - boto.ec2.instance.Instances provisioned
 104         """
 105         vpc = boto.vpc.VPCConnection()
 106         subnet = vpc.get_all_subnets(subnet_ids=__default_subnet__)[0]
 107         cidr_block = subnet.cidr_block
 108         vpc.close()
 109
 110         user_data = "#cloud-config\n"
 111         user_data += "runcmd:\n"
 112         user_data += " - echo '/home %s(rw,root_squash,sync)' >  /etc/exports\n" % (cidr_block,)
 113         user_data += " - systemctl start nfs-server\n"
 114         user_data += " - systemctl enable nfs-server\n"
 115         user_data += self.cloud_init_yum_repos()
 116         user_data += self.cloud_init_current_user()
 117
 118
 119         net_if = boto.ec2.networkinterface.NetworkInterfaceSpecification(
 120                 subnet_id=__default_subnet__,
 121                 groups=[__default_security_group__,],
 122                 associate_public_ip_address=True)
 123
 124         net_ifs = boto.ec2.networkinterface.NetworkInterfaceCollection(net_if)
 125
 126         new_reservation = self._conn.run_instances(
 127                 image_id=self._ami,
 128                 min_count=1,
 129                 max_count=1,
 130                 instance_type=__default_instance_type__,
 131                 network_interfaces=net_ifs,
 132                 tenancy='default',
 133                 user_data=user_data)
 134         instance = new_reservation.instances[0]
 135
 136         instance.add_tag('parent_component', cluster_component)
 137         instance.add_tag('parent_instance', cluster_instance)
 138         instance.add_tag('master', 'self')
 139
 140         return instance
 141
 142
 143     def provision(self, cluster_component, cluster_instance, n_instances=1, master_instance=None, net_ifs=None):
 144         """
 145         Provision a number of EC2 instanced to be used in a cluster.
 146
 147         @param cluster_component  - parent cluster of each instance
 148         @param cluster_instance   - instance id of the owning cluster
 149         @param n_instances        - number of requested instances
 150         @param master_instance    - if specified, the boto.ec2.instance.Instance that is providing master
 151                                     services for this cluster
 152
 153         @return                   - list of boto.ec2.instance.Instances provisioned
 154         """
 155         instances = []
 156         cluster_instance = int(cluster_instance)
 157
 158         def posess_instance(instance):
 159             instances.append(instance)
 160             instance.add_tag('parent_component', cluster_component)
 161             instance.add_tag('parent_instance', cluster_instance)
 162             if master_instance is not None:
 163                 instance.add_tag('master', master_instance.id)
 164             else:
 165                 instance.add_tag('master', 'None')
 166
 167         user_data = "#cloud-config\n"
 168         user_data += self.cloud_init_current_user()
 169         user_data += "runcmd:\n"
 170         user_data += self.cloud_init_yum_repos()
 171
 172         if master_instance is not None:
 173             user_data += " - echo '%s:/home /home nfs rw,soft,sync 0 0' >> /etc/fstab\n" % (
 174                     master_instance.private_ip_address,)
 175             user_data += " - mount /home\n"
 176
 177         if net_ifs is not None:
 178             kwds = {'subnet_id': __default_subnet__}
 179         else:
 180             kwds = {'network_interfaces': net_ifs}
 181             print net_ifs
 182
 183         new_reservation = self._conn.run_instances(
 184             image_id=self._ami,
 185             min_count=n_instances,
 186             max_count=n_instances,
 187             instance_type=__default_instance_type__,
 188             tenancy='default',
 189             user_data=user_data,
 190             network_interfaces=net_ifs)
 191
 192         _ = [posess_instance(i) for i in new_reservation.instances]
 193
 194         return instances
 195
 196     def stop(self, instance_id, free_resources=True):
 197         """
 198         Stop the specified instance, freeing all allocated resources (elastic ips, etc) if requested.
 199
 200         @param instance_id      - name of the instance to stop
 201         @param free_resource    - If True that all resources that were only owned by this instance
 202                                   will be deallocated as well.
 203         """
 204         self._conn.terminate_instances(instance_ids=[instance_id,])
 205
 206     def fastpath111(self):
 207         vpc_conn = boto.vpc.VPCConnection()
 208         vpc = vpc_conn.get_all_vpcs(vpc_ids=[__default_vpc__,])[0]
 209         subnet_addrs_split = vpc.cidr_block.split('.')
 210
 211         networks = {
 212             'mgmt': [s for s in vpc_conn.get_all_subnets() if s.id == __default_subnet__][0],
 213             'tg_fabric': None,
 214             'ts_fabric': None,
 215             'tg_lb_ext': None,
 216             'lb_ts_ext': None,
 217         }
 218
 219         for i, network in enumerate([n for n, s in networks.items() if s == None]):
 220             addr = "%s.%s.10%d.0/25" % (subnet_addrs_split[0], subnet_addrs_split[1], i)
 221             try:
 222                 subnet = vpc_conn.create_subnet(vpc.id, addr)
 223             except boto.exception.EC2ResponseError, e:
 224                 if 'InvalidSubnet.Conflict' == e.error_code:
 225                     subnet = vpc_conn.get_all_subnets(filters=[('vpcId', vpc.id), ('cidrBlock', addr)])[0]
 226                 else:
 227                     raise
 228
 229             networks[network] = subnet
 230
 231         def create_interfaces(nets):
 232             ret = boto.ec2.networkinterface.NetworkInterfaceCollection()
 233
 234             for i, network in enumerate(nets):
 235                 spec = boto.ec2.networkinterface.NetworkInterfaceSpecification(
 236                         subnet_id=networks[network].id,
 237                         description='%s iface' % (network,),
 238                         groups=[__default_security_group__],
 239                         device_index=i)
 240                 ret.append(spec)
 241
 242             return ret
 243
 244         ret = {}
 245
 246         ret['cli'] = self.provision_master('fp111', 1)
 247         ret['cli'].add_tag('Name', 'cli')
 248
 249         net_ifs = create_interfaces(['mgmt'])
 250         ret['mgmt'] = self.provision('fp111', 1, master_instance=ret['cli'], net_ifs=net_ifs)[0]
 251         ret['mgmt'].add_tag('Name', 'mgmt')
 252
 253         net_ifs = create_interfaces(['mgmt', 'tg_fabric'])
 254         ret['tg1'] = self.provision('fp111', 1, master_instance=ret['cli'], net_ifs=net_ifs)[0]
 255         ret['tg1'].add_tag('Name', 'tg1')
 256
 257         net_ifs = create_interfaces(['mgmt', 'tg_fabric', 'tg_lb_ext'])
 258         ret['tg2'] = self.provision('fp111', 1, master_instance=ret['cli'], net_ifs=net_ifs)[0]
 259         ret['tg2'].add_tag('Name', 'tg2')
 260
 261         net_ifs = create_interfaces(['mgmt', 'ts_fabric'])
 262         ret['ts1'] = self.provision('fp111', 1, master_instance=ret['cli'], net_ifs=net_ifs)[0]
 263         ret['ts1'].add_tag('Name', 'ts1')
 264
 265         net_ifs = create_interfaces(['mgmt', 'ts_fabric', 'lb_ts_ext'])
 266         ret['ts3'] = self.provision('fp111', 1, master_instance=ret['cli'], net_ifs=net_ifs)[0]
 267         ret['ts3'].add_tag('Name', 'ts3')
 268
 269         net_ifs = create_interfaces(['mgmt', 'ts_fabric', 'lb_ts_ext', 'tg_lb_ext'])
 270         ret['ts2'] = self.provision('fp111', 1, master_instance=ret['cli'], net_ifs=net_ifs)[0]
 271         ret['ts2'].add_tag('Name', 'ts2')
 272
 273         return ret
 274
 275 # vim: sw=4