rwcal/test/ec2.py

# 
#   Copyright 2016 RIFT.IO Inc
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#

import glob
import itertools
import os

import boto
import boto.vpc

# TODO:  Pull the lastest of owned instances.
__default_instance_ami__ = 'ami-e421bc8c'

# TODO:  Make VPC's per user?
__default_subnet__ = 'subnet-4b484363'
__default_security_group__ = 'sg-d9da90bc'

__default_instance_type__ = 'm1.medium'
__default_vpc__ = 'vpc-e7ed4482'

class RWEC2(object):
    def __init__(self,  subnet=None, ami=None):
        self._subnet = subnet if subnet is not None else __default_subnet__
        self._ami = ami if ami is not None else __default_instance_ami__

        self._conn = boto.connect_ec2()

    @staticmethod
    def cloud_init_current_user():
        """
        Return user_data configuration suitable for cloud-init that will create a user
        with sudo and ssh key access on the remote instance.

        ssh keys are found with the glob ~/.ssh/*pub*
        """
        user_data = "users:\n"
        user_data += " - name: %s\n" % (os.getlogin(),)
        user_data += "   groups: [wheel, adm, systemd-journal]\n"
        user_data += "   sudo: [\"ALL=(ALL) NOPASSWD:ALL\"]\n"
        user_data += "   shell: /bin/bash\n"
        user_data += "   ssh_authorized_keys:\n"
        for pub_key in glob.glob('%s/.ssh/*pub*' % (os.environ['HOME'],)):
            with open(pub_key) as fp:
                user_data += "    -  %s" % (fp.read(),)

        return user_data


    @staticmethod
    def cloud_init_yum_repos():
        """
        Return a string of user_data commands that can be used to update the yum
        repos to point to the correct location.  They should be added by the caller
        within a 'runcmd:' block.
        """
        ret = " - sed -i -e 's,www\.,,' -e 's,riftio\.com/mirrors,riftio.com:8881,' /etc/yum.repos.d/*.repo\n"
        return ret

    def instances(self, cluster_component, cluster_instance):
        """
        List of instances owned by the given cluster instance

        @param cluster_component  - parent cluster of each instance
        @param cluster_instance   - instance id of the owning cluster
        @param n_instances        - number of requested instances

        @return                   - list of boto.ec2.instance.Instances provisioned
        """
        ret = []
        reservations = self._conn.get_all_instances()
        for instance in [instance for reservation in reservations for instance in reservation.instances]:
            tags = instance.tags
            if (tags.get('parent_component') == cluster_component
                    and tags.get('parent_instance') == cluster_instance):
                ret.append(instance)

        return ret

    def provision_master(self, cluster_component, cluster_instance):
        """
        Provision a master instance in EC2.  The master instance is a special instance with the
        following features:
            - Public IP
            - /home shared over NFS

        @param cluster_component  - parent cluster of each instance
        @param cluster_instance   - instance id of the owning cluster

        @return                   - boto.ec2.instance.Instances provisioned
        """
        vpc = boto.vpc.VPCConnection()
        subnet = vpc.get_all_subnets(subnet_ids=__default_subnet__)[0]
        cidr_block = subnet.cidr_block
        vpc.close()

        user_data = "#cloud-config\n"
        user_data += "runcmd:\n"
        user_data += " - echo '/home %s(rw,root_squash,sync)' >  /etc/exports\n" % (cidr_block,)
        user_data += " - systemctl start nfs-server\n"
        user_data += " - systemctl enable nfs-server\n"
        user_data += self.cloud_init_yum_repos()
        user_data += self.cloud_init_current_user()


        net_if = boto.ec2.networkinterface.NetworkInterfaceSpecification(
                subnet_id=__default_subnet__,
                groups=[__default_security_group__,],
                associate_public_ip_address=True)

        net_ifs = boto.ec2.networkinterface.NetworkInterfaceCollection(net_if)

        new_reservation = self._conn.run_instances(
                image_id=self._ami,
                min_count=1,
                max_count=1,
                instance_type=__default_instance_type__,
                network_interfaces=net_ifs,
                tenancy='default',
                user_data=user_data)
        instance = new_reservation.instances[0]

        instance.add_tag('parent_component', cluster_component)
        instance.add_tag('parent_instance', cluster_instance)
        instance.add_tag('master', 'self')

        return instance


    def provision(self, cluster_component, cluster_instance, n_instances=1, master_instance=None, net_ifs=None):
        """
        Provision a number of EC2 instanced to be used in a cluster.

        @param cluster_component  - parent cluster of each instance
        @param cluster_instance   - instance id of the owning cluster
        @param n_instances        - number of requested instances
        @param master_instance    - if specified, the boto.ec2.instance.Instance that is providing master
                                    services for this cluster

        @return                   - list of boto.ec2.instance.Instances provisioned
        """
        instances = []
        cluster_instance = int(cluster_instance)

        def posess_instance(instance):
            instances.append(instance)
            instance.add_tag('parent_component', cluster_component)
            instance.add_tag('parent_instance', cluster_instance)
            if master_instance is not None:
                instance.add_tag('master', master_instance.id)
            else:
                instance.add_tag('master', 'None')

        user_data = "#cloud-config\n"
        user_data += self.cloud_init_current_user()
        user_data += "runcmd:\n"
        user_data += self.cloud_init_yum_repos()

        if master_instance is not None:
            user_data += " - echo '%s:/home /home nfs rw,soft,sync 0 0' >> /etc/fstab\n" % (
                    master_instance.private_ip_address,)
            user_data += " - mount /home\n"

        if net_ifs is not None:
            kwds = {'subnet_id': __default_subnet__}
        else:
            kwds = {'network_interfaces': net_ifs}
            print net_ifs

        new_reservation = self._conn.run_instances(
            image_id=self._ami,
            min_count=n_instances,
            max_count=n_instances,
            instance_type=__default_instance_type__,
            tenancy='default',
            user_data=user_data,
            network_interfaces=net_ifs)

        _ = [posess_instance(i) for i in new_reservation.instances]

        return instances

    def stop(self, instance_id, free_resources=True):
        """
        Stop the specified instance, freeing all allocated resources (elastic ips, etc) if requested.

        @param instance_id      - name of the instance to stop
        @param free_resource    - If True that all resources that were only owned by this instance
                                  will be deallocated as well.
        """
        self._conn.terminate_instances(instance_ids=[instance_id,])

    def fastpath111(self):
        vpc_conn = boto.vpc.VPCConnection()
        vpc = vpc_conn.get_all_vpcs(vpc_ids=[__default_vpc__,])[0]
        subnet_addrs_split = vpc.cidr_block.split('.')

        networks = {
            'mgmt': [s for s in vpc_conn.get_all_subnets() if s.id == __default_subnet__][0],
            'tg_fabric': None,
            'ts_fabric': None,
            'tg_lb_ext': None,
            'lb_ts_ext': None,
        }

        for i, network in enumerate([n for n, s in networks.items() if s == None]):
            addr = "%s.%s.10%d.0/25" % (subnet_addrs_split[0], subnet_addrs_split[1], i)
            try:
                subnet = vpc_conn.create_subnet(vpc.id, addr)
            except boto.exception.EC2ResponseError, e:
                if 'InvalidSubnet.Conflict' == e.error_code:
                    subnet = vpc_conn.get_all_subnets(filters=[('vpcId', vpc.id), ('cidrBlock', addr)])[0]
                else:
                    raise

            networks[network] = subnet

        def create_interfaces(nets):
            ret = boto.ec2.networkinterface.NetworkInterfaceCollection()

            for i, network in enumerate(nets):
                spec = boto.ec2.networkinterface.NetworkInterfaceSpecification(
                        subnet_id=networks[network].id,
                        description='%s iface' % (network,),
                        groups=[__default_security_group__],
                        device_index=i)
                ret.append(spec)

            return ret

        ret = {}

        ret['cli'] = self.provision_master('fp111', 1)
        ret['cli'].add_tag('Name', 'cli')

        net_ifs = create_interfaces(['mgmt'])
        ret['mgmt'] = self.provision('fp111', 1, master_instance=ret['cli'], net_ifs=net_ifs)[0]
        ret['mgmt'].add_tag('Name', 'mgmt')

        net_ifs = create_interfaces(['mgmt', 'tg_fabric'])
        ret['tg1'] = self.provision('fp111', 1, master_instance=ret['cli'], net_ifs=net_ifs)[0]
        ret['tg1'].add_tag('Name', 'tg1')

        net_ifs = create_interfaces(['mgmt', 'tg_fabric', 'tg_lb_ext'])
        ret['tg2'] = self.provision('fp111', 1, master_instance=ret['cli'], net_ifs=net_ifs)[0]
        ret['tg2'].add_tag('Name', 'tg2')

        net_ifs = create_interfaces(['mgmt', 'ts_fabric'])
        ret['ts1'] = self.provision('fp111', 1, master_instance=ret['cli'], net_ifs=net_ifs)[0]
        ret['ts1'].add_tag('Name', 'ts1')

        net_ifs = create_interfaces(['mgmt', 'ts_fabric', 'lb_ts_ext'])
        ret['ts3'] = self.provision('fp111', 1, master_instance=ret['cli'], net_ifs=net_ifs)[0]
        ret['ts3'].add_tag('Name', 'ts3')

        net_ifs = create_interfaces(['mgmt', 'ts_fabric', 'lb_ts_ext', 'tg_lb_ext'])
        ret['ts2'] = self.provision('fp111', 1, master_instance=ret['cli'], net_ifs=net_ifs)[0]
        ret['ts2'].add_tag('Name', 'ts2')

        return ret

# vim: sw=4