X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;ds=sidebyside;f=rwcal%2Ftest%2Fec2.py;fp=rwcal%2Ftest%2Fec2.py;h=59ad04939acc1246adcca69c35d01335357c9dbf;hb=6f07e6f33f751ab4ffe624f6037f887b243bece2;hp=0000000000000000000000000000000000000000;hpb=72a563886272088feb7cb52e4aafbe6d2c580ff9;p=osm%2FSO.git diff --git a/rwcal/test/ec2.py b/rwcal/test/ec2.py new file mode 100644 index 00000000..59ad0493 --- /dev/null +++ b/rwcal/test/ec2.py @@ -0,0 +1,275 @@ + +# +# Copyright 2016 RIFT.IO Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import glob +import itertools +import os + +import boto +import boto.vpc + +# TODO: Pull the lastest of owned instances. +__default_instance_ami__ = 'ami-e421bc8c' + +# TODO: Make VPC's per user? +__default_subnet__ = 'subnet-4b484363' +__default_security_group__ = 'sg-d9da90bc' + +__default_instance_type__ = 'm1.medium' +__default_vpc__ = 'vpc-e7ed4482' + +class RWEC2(object): + def __init__(self, subnet=None, ami=None): + self._subnet = subnet if subnet is not None else __default_subnet__ + self._ami = ami if ami is not None else __default_instance_ami__ + + self._conn = boto.connect_ec2() + + @staticmethod + def cloud_init_current_user(): + """ + Return user_data configuration suitable for cloud-init that will create a user + with sudo and ssh key access on the remote instance. + + ssh keys are found with the glob ~/.ssh/*pub* + """ + user_data = "users:\n" + user_data += " - name: %s\n" % (os.getlogin(),) + user_data += " groups: [wheel, adm, systemd-journal]\n" + user_data += " sudo: [\"ALL=(ALL) NOPASSWD:ALL\"]\n" + user_data += " shell: /bin/bash\n" + user_data += " ssh_authorized_keys:\n" + for pub_key in glob.glob('%s/.ssh/*pub*' % (os.environ['HOME'],)): + with open(pub_key) as fp: + user_data += " - %s" % (fp.read(),) + + return user_data + + + @staticmethod + def cloud_init_yum_repos(): + """ + Return a string of user_data commands that can be used to update the yum + repos to point to the correct location. They should be added by the caller + within a 'runcmd:' block. + """ + ret = " - sed -i -e 's,www\.,,' -e 's,riftio\.com/mirrors,riftio.com:8881,' /etc/yum.repos.d/*.repo\n" + return ret + + def instances(self, cluster_component, cluster_instance): + """ + List of instances owned by the given cluster instance + + @param cluster_component - parent cluster of each instance + @param cluster_instance - instance id of the owning cluster + @param n_instances - number of requested instances + + @return - list of boto.ec2.instance.Instances provisioned + """ + ret = [] + reservations = self._conn.get_all_instances() + for instance in [instance for reservation in reservations for instance in reservation.instances]: + tags = instance.tags + if (tags.get('parent_component') == cluster_component + and tags.get('parent_instance') == cluster_instance): + ret.append(instance) + + return ret + + def provision_master(self, cluster_component, cluster_instance): + """ + Provision a master instance in EC2. The master instance is a special instance with the + following features: + - Public IP + - /home shared over NFS + + @param cluster_component - parent cluster of each instance + @param cluster_instance - instance id of the owning cluster + + @return - boto.ec2.instance.Instances provisioned + """ + vpc = boto.vpc.VPCConnection() + subnet = vpc.get_all_subnets(subnet_ids=__default_subnet__)[0] + cidr_block = subnet.cidr_block + vpc.close() + + user_data = "#cloud-config\n" + user_data += "runcmd:\n" + user_data += " - echo '/home %s(rw,root_squash,sync)' > /etc/exports\n" % (cidr_block,) + user_data += " - systemctl start nfs-server\n" + user_data += " - systemctl enable nfs-server\n" + user_data += self.cloud_init_yum_repos() + user_data += self.cloud_init_current_user() + + + net_if = boto.ec2.networkinterface.NetworkInterfaceSpecification( + subnet_id=__default_subnet__, + groups=[__default_security_group__,], + associate_public_ip_address=True) + + net_ifs = boto.ec2.networkinterface.NetworkInterfaceCollection(net_if) + + new_reservation = self._conn.run_instances( + image_id=self._ami, + min_count=1, + max_count=1, + instance_type=__default_instance_type__, + network_interfaces=net_ifs, + tenancy='default', + user_data=user_data) + instance = new_reservation.instances[0] + + instance.add_tag('parent_component', cluster_component) + instance.add_tag('parent_instance', cluster_instance) + instance.add_tag('master', 'self') + + return instance + + + def provision(self, cluster_component, cluster_instance, n_instances=1, master_instance=None, net_ifs=None): + """ + Provision a number of EC2 instanced to be used in a cluster. + + @param cluster_component - parent cluster of each instance + @param cluster_instance - instance id of the owning cluster + @param n_instances - number of requested instances + @param master_instance - if specified, the boto.ec2.instance.Instance that is providing master + services for this cluster + + @return - list of boto.ec2.instance.Instances provisioned + """ + instances = [] + cluster_instance = int(cluster_instance) + + def posess_instance(instance): + instances.append(instance) + instance.add_tag('parent_component', cluster_component) + instance.add_tag('parent_instance', cluster_instance) + if master_instance is not None: + instance.add_tag('master', master_instance.id) + else: + instance.add_tag('master', 'None') + + user_data = "#cloud-config\n" + user_data += self.cloud_init_current_user() + user_data += "runcmd:\n" + user_data += self.cloud_init_yum_repos() + + if master_instance is not None: + user_data += " - echo '%s:/home /home nfs rw,soft,sync 0 0' >> /etc/fstab\n" % ( + master_instance.private_ip_address,) + user_data += " - mount /home\n" + + if net_ifs is not None: + kwds = {'subnet_id': __default_subnet__} + else: + kwds = {'network_interfaces': net_ifs} + print net_ifs + + new_reservation = self._conn.run_instances( + image_id=self._ami, + min_count=n_instances, + max_count=n_instances, + instance_type=__default_instance_type__, + tenancy='default', + user_data=user_data, + network_interfaces=net_ifs) + + _ = [posess_instance(i) for i in new_reservation.instances] + + return instances + + def stop(self, instance_id, free_resources=True): + """ + Stop the specified instance, freeing all allocated resources (elastic ips, etc) if requested. + + @param instance_id - name of the instance to stop + @param free_resource - If True that all resources that were only owned by this instance + will be deallocated as well. + """ + self._conn.terminate_instances(instance_ids=[instance_id,]) + + def fastpath111(self): + vpc_conn = boto.vpc.VPCConnection() + vpc = vpc_conn.get_all_vpcs(vpc_ids=[__default_vpc__,])[0] + subnet_addrs_split = vpc.cidr_block.split('.') + + networks = { + 'mgmt': [s for s in vpc_conn.get_all_subnets() if s.id == __default_subnet__][0], + 'tg_fabric': None, + 'ts_fabric': None, + 'tg_lb_ext': None, + 'lb_ts_ext': None, + } + + for i, network in enumerate([n for n, s in networks.items() if s == None]): + addr = "%s.%s.10%d.0/25" % (subnet_addrs_split[0], subnet_addrs_split[1], i) + try: + subnet = vpc_conn.create_subnet(vpc.id, addr) + except boto.exception.EC2ResponseError, e: + if 'InvalidSubnet.Conflict' == e.error_code: + subnet = vpc_conn.get_all_subnets(filters=[('vpcId', vpc.id), ('cidrBlock', addr)])[0] + else: + raise + + networks[network] = subnet + + def create_interfaces(nets): + ret = boto.ec2.networkinterface.NetworkInterfaceCollection() + + for i, network in enumerate(nets): + spec = boto.ec2.networkinterface.NetworkInterfaceSpecification( + subnet_id=networks[network].id, + description='%s iface' % (network,), + groups=[__default_security_group__], + device_index=i) + ret.append(spec) + + return ret + + ret = {} + + ret['cli'] = self.provision_master('fp111', 1) + ret['cli'].add_tag('Name', 'cli') + + net_ifs = create_interfaces(['mgmt']) + ret['mgmt'] = self.provision('fp111', 1, master_instance=ret['cli'], net_ifs=net_ifs)[0] + ret['mgmt'].add_tag('Name', 'mgmt') + + net_ifs = create_interfaces(['mgmt', 'tg_fabric']) + ret['tg1'] = self.provision('fp111', 1, master_instance=ret['cli'], net_ifs=net_ifs)[0] + ret['tg1'].add_tag('Name', 'tg1') + + net_ifs = create_interfaces(['mgmt', 'tg_fabric', 'tg_lb_ext']) + ret['tg2'] = self.provision('fp111', 1, master_instance=ret['cli'], net_ifs=net_ifs)[0] + ret['tg2'].add_tag('Name', 'tg2') + + net_ifs = create_interfaces(['mgmt', 'ts_fabric']) + ret['ts1'] = self.provision('fp111', 1, master_instance=ret['cli'], net_ifs=net_ifs)[0] + ret['ts1'].add_tag('Name', 'ts1') + + net_ifs = create_interfaces(['mgmt', 'ts_fabric', 'lb_ts_ext']) + ret['ts3'] = self.provision('fp111', 1, master_instance=ret['cli'], net_ifs=net_ifs)[0] + ret['ts3'].add_tag('Name', 'ts3') + + net_ifs = create_interfaces(['mgmt', 'ts_fabric', 'lb_ts_ext', 'tg_lb_ext']) + ret['ts2'] = self.provision('fp111', 1, master_instance=ret['cli'], net_ifs=net_ifs)[0] + ret['ts2'].add_tag('Name', 'ts2') + + return ret + +# vim: sw=4