ec2-backup

#!/bin/bash
###################################
# ec2 backup, a backup tool using AWS
#
# Authors:
# Eli Davis <edavis1@stevens.edu>
# Paul-Anthony Dudzinski <pdudzins@stevens.edu
# Nick Noga <nnoga@stevens.edu>
#
#(c) 2014 by The Authors
#
#
###################################

#Vars
#####################
f_debug=0
f_uservol=0
method="dd"
backup_name="backup.tar"

###VOLUME
volume=""
volumeStatus=""
volumeSize=""

###INSTANCE
image="ami-6de0dd04"
ec2defaults="--image-id $image  --count 1 --instance-type t1.micro"
username="ubuntu"
sshflags=""
instance=""
hostname=""
sshPort=22
timeout=5
ncOutput=""
instanceState=""
volzone=""
zone=""

attachment=""
attachStatus=""
attachPoint="/dev/sdh"
devicename="/dev/xvdh"
mountpoint="/media/backup"
rsyncstatus=""

#Functions
#####################
usage() {
echo "ec2_backup [-h] [-m method] [-v volume-id] dir "
} 


errorTerminate() {
        if [ ! -z "$instance" ]; then
                aws ec2 terminate-instances --instance-ids $instance >/dev/null
        fi

        if [ ! -z "$volume" ] && [ $f_uservol -eq 0 ] ; then
		aws ec2 detach-volume --force --volume-id $volume --instance-id $instance >/dev/null
                aws ec2 delete-volume --volume-id $volume >/dev/null
        fi

}


#get the size and status of the volume
#the volume id is passed as the first and only argument
getVolumeStatus() {
volumeStatus=`aws --output text ec2 describe-volumes --volume-ids $1|awk '{print$6}'`
volumeSize=`aws --output text ec2 describe-volumes --volume-ids $1|awk '{print$4}'`

#if the volume does not come back in one of these two state it is unusable by out program
if [ "$volumeStatus" != "available" ] && [ "$volumeStatus" != "creating" ];then
	#echo "$volumeStatus"
	echo "The volume is in use, deleted, or there was an error creating it" 1>&2
	errorTerminate
	exit 1
fi
}


#Verifies that default image is available. Exits if not.
verifyImageAvailability() {
imageout=`aws ec2 describe-images --image-ids $image`
if [[ $imageout == *InvalidAMIID* ]]; then
	echo "The default image is not available to create an instance."
	exit 1
fi
}

#Finds an availability zone for the user. Finds volume zone if -v specified.
getAvailabilityZone() {

if [ ! -z $volume]; then
	zone="--region `aws ec2 describe-volumes --output text | grep $volume | awk '{print $2}'`"
else
	zone=`aws ec2 describe-availability-zones --output text | grep -m 1 "1" | awk '{print $4}'`
fi
}


#wait for the volume to become available
#there are no arguments for this function, it is dependent on global vars
volumeWait() {
	if [ $f_debug -eq 1 ]; then
		echo "Volume initializing..."
	fi
        while [[ "$volumeStatus" != *"available"* ]]; do
                getVolumeStatus $volume
		sleep 1
		if [ $f_debug -eq 1 ]; then
                echo "Waiting for the volume to be available..."
        	fi

        done
}

#This function should be passed the backup directory as the only argument
#the volume is set and checked for existence in the arg parser
#the global zone variable must be set, this can be done by the user environment or by the
#function that will create the instance 
getVolume() {

#Determine the size of the file and translate it into GB to run the command
bytesize=$(du -sb $1 2>/dev/null | cut -f1)
#this function produces a float and outputs 0 if > 1024 bytes
gbsize=$(python -c "from math import ceil; print ceil(ceil(ceil($bytesize/1024)/1024)/1024)")
#Convert gbsize to integer
gbsize=${gbsize/.*}
alloc=0

## Determine and allocate necessary volume size IF a volume was not provided
if [ -z $volume ]; then
        if [ $gbsize -eq 1 ] || [ $gbsize -eq 0 ]; then ## Minimum volume allocation is 1GB
                ## Allocate 2GB
                alloc=2
        elif [ $gbsize -lt 513 ] && [ $gbsize -gt 1 ]; then ## Maximum volume allocation is 1024GB
                ## Allocate 2*gbsize
                 let alloc=gbsize*2
        else
                echo "Directory is too large to be allocated into a single volume." 1>&2
                exit 1
        fi
	if [ -z $alloc ] || [ $alloc -eq 0 ]; then
		echo "Invalid allocation size" 1>&2
		exit 1
	fi

	volume=$(aws --output text ec2 create-volume --size $alloc --availability-zone $zone | awk '{print $6}')
	getVolumeStatus $volume
	#wait for the volume to be available busy wait....
	volumeWait 	
else
##volume entered

	getVolumeStatus $volume
	#Only error here I can think of outside status and size is zone incompatibility
	if [ $gbsize -gt $volumeSize ]; then
		echo "The volume given is too small to backup the file" 1>&2
                exit 1
	elif [ $gbsize -eq $volumeSize ] && [ $gbsize -ne 1 ]; then
		echo "The volume size and directory size are equal use a larger volume" 1>&2
		exit 1
	fi

	volumeWait
fi

}

#This function is for parsing the environment variables that need to be set for the 
#program to work correctly. We need to look a the two that pertain to EC2 as per the spec
#not sure what I can do to make the AMAZON environment variables work correctly...
getEnvironmentVars() {

if [ $f_debug -eq 1 ]; then
                echo "Checking for environment variables..."
fi


ec2flags=${EC2_BACKUP_FLAGS_AWS}
verbose=${EC2_BACKUP_VERBOSE}

#Parse the keypair from the variable on whitespace
f_keypair=0
f_secgroup=0
secgroup=""
keypair=""

#get the two arguments if they exist
args=($EC2_BACKUP_FLAGS_AWS)
for ((index=0; index <= ${#args[@]}; index++));
do
         if [ "${args[index]}" == "--security-groups" ]; then
                 secgroup=${args[index+1]}
         fi
         if [ "${args[index]}" == "--key-name" ]; then
                keypair=${args[index+1]}
         fi

done


if [ $f_debug -eq 1 ]; then
echo "Security Group: $secgroup  Kenyname: $keypair"
fi


#If either the keypair or the security group are null then exit 1
if [ -z  $keypair ] || [  -z $secgroup  ]; then
        echo "The keypair or the security group are empty please set them in EC2_BACKUP_FLAGS_AWS" 1>&2
        exit 1
fi


#ask aws for the security groups and see if the input in the variable is in the list
for group in $( aws --output text  ec2 describe-security-groups |
		 awk -F"\t" '{if ($1 == "SECURITYGROUPS"){print $4}}')
do
	if [ "$group" = "$secgroup" ]; then
		f_secgroup=1
		break
	fi
done

# if the flag is equal 0 it was not in the list above, exit 1 b/c there is not a valid security group
if [ $f_secgroup -eq 0 ]; then
	echo "The security group that was supplied is not valid" 1>&2
	exit 1
fi

#check the keypairs in the users setup and see if the keypair passed is in them
for key in $( aws --output text ec2 describe-key-pairs | awk -F"\t" '{print $3}')
do

	if [ "$key" = "$keypair" ]; then
		f_keypair=1
		break
	fi
done

# if the flag is equal 0 it was not in the list above, exit 1 b/c there is not a valid security group
if [ $f_keypair -eq 0 ]; then
        echo "The keypair that was supplied is not valid" 1>&2
        exit 1
fi


if [ ! -z "$verbose" ];then
	f_debug=1;
fi

}



###### ATTACHMENT FUNCTIONS
#This function attaches a given volume and a given instance and ensures that the
#volume has been attached prior to returning. It attaches to /dev/sdh which should be
#empty on a new instance. This is inside the scope of the program as no pre-existing
#instances are to be used.
attachVolume() {

if [ $f_debug -eq 1 ]; then
        echo "Attaching volume $volume to instance $instance..."
fi

attachment=$(aws ec2 attach-volume --volume-id $volume --instance-id $instance --device $attachPoint)
attachWait
if [ $f_debug -eq 1 ]; then
        echo "Volume $volume has successfully been attached."
fi

}

#Waits for attachment to say "attached"
attachWait() {
        if [ $f_debug -eq 1 ]; then
                echo "Waiting for attachment..."
        fi
        attachStatus=`aws ec2 describe-volumes --output text | grep $attachPoint | awk '{print $6}'`

        #Loop until attached
        while [[ "$attachStatus" != *"attached"* ]]; do

                sleep 1

                attachStatus=`aws ec2 describe-volumes --output text | grep $attachPoint | awk '{print $6}'`
        done
}

#this function waits for the instance to return running so that we can wait for ssh
#this function will also terminate on any instance status code besides pending and running
getInstanceStatus() {
instanceState=`aws ec2 describe-instances --output text --instance-id $instance | grep STATE| awk '{print $3}'`

        if [ "$instanceState" != "pending" ] && [ "$instanceState" != "running" ]; then
                echo " Instance is in an unusable state" 1>&2
		errorTerminate
                exit 1;
        fi
}


#This function gets the instance and will exit if the intance cannot be run as per the
#backupflags that were supplied. It will then call the getInstanceStatus function to
#busywait until the intance is in the running state to prepare it for the ssh testing
getInstance(){
dryrun_success="A client error (DryRunOperation) occurred when calling the RunInstances operation: Request would have succeeded, but DryRun flag is set."


instance_dryrun=`aws ec2  run-instances --output text --dry-run --placement AvailabilityZone=$zone $ec2defaults ${EC2_BACKUP_FLAGS_AWS} 2>&1`

if [ "$instance_dryrun" != "$dryrun_success" ]; then
        echo "Instance instantiation failed : $instance_dryrun" 1>&2
	errorTerminate
        exit 1;
fi

instance=`aws ec2 run-instances --output text --placement AvailabilityZone=$zone $ec2defaults ${EC2_BACKUP_FLAGS_AWS}|grep INSTANCES | awk '{print $8}'`


if [ $f_debug -eq 1 ]; then
        echo "Waiting for the instance to be available..."
fi

while [[ "$instanceState" != "running" ]]; do
                getInstanceStatus
                sleep 1
        done

if [ $f_debug -eq 1 ]; then
	echo "Instance ID: $instance "
fi
#Extra sleep to make sure that the instance has more time to boot up before ssh wait happends
sleep 3
}

terminateInstance() {

if [ $f_debug -eq 1 ]; then
        echo "Cleaning up instnce: $instance "
fi

aws ec2 terminate-instances --instance-ids $instance >/dev/null
}


getHostname(){

if [ $f_debug -eq 1 ]; then
        echo "Waiting to get hostname..."
fi

while [ -z $hostname ] || [[ "$hostname" == *"None"* ]]; do
	hostname=`aws ec2 describe-instances --output text | grep $instance | awk '{print $15}'`
done	

if [ $f_debug -eq 1 ]; then
        echo "Hostname found: $hostname"
fi

}

#checkSSH will check the readiness of the remote host for an SSH connection.
checkSSH(){

if [ $f_debug -eq 1 ]; then
        echo "Checking Host Availability..."
fi


while [[ "$ncOutput" != *"succeeded"* ]]; do
	ncOutput=`nc -zv -w $timeout $hostname $sshPort 2>&1`
done

if [ $f_debug -eq 1 ]; then
        echo "Host Up!"
fi


}



#DD BACKUP
#This function will archive the specified directory and backup to the remote location using dd
#It will take the directory to be backed up as its only argument
####################
ddBackup(){

if [ $f_debug -eq 1 ]; then
        echo "dd to remote..."
fi

var=`tar -cf - $1 | ssh \${EC2_BACKUP_FLAGS_SSH} -o StrictHostKeyChecking=no $username@$hostname 'sudo dd of=devicename' 2>&1`
if [ $? -gt 0 ];then
	echo $var 1>&2
	errorTerminate
        exit 1
fi


if [ $f_debug -eq 1 ]; then
        echo "dd complete"
fi
}

#Rsync backup
rsyncBackup() {
if [ $f_debug -eq 1 ]; then
	echo "Performing rsync on directory..."
fi

rsyncstatus=`rsync -ravv --rsync-path="sudo rsync" -e "ssh $EC2_BACKUP_FLAGS_SSH -o StrictHostKeyChecking=no" --delete $1 $username@$hostname:$mountpoint 2>&1`

if [ $? -gt 0 ]; then
	echo $rsyncstat 1>&2
	errorTerminate
	exit 1
fi

if [ $f_debug -eq 1 ]; then
	echo "rsync complete."
fi
}

#createFilesystem
#this function will create a filesystem on the attached volume and mount the 
createFilesystem(){

if [ $f_debug -eq 1 ]; then
        echo "Creating filesystem on volume..."
fi


var=`ssh  -o "StrictHostKeyChecking=no" ${EC2_BACKUP_FLAGS_SSH}  ubuntu@$hostname "
sudo mkfs -t ext4 $devicename >/dev/null 2>&1
sudo mkdir $mountpoint >/dev/null 2>&1
sudo mount $devicename $mountpoint >/dev/null 2>&1
exit
" 2>&1`

if [ $? -gt 0 ]; then
	echo "Failed to SSH to create filesystem: $var" 1>&2;
	errorTerminate
	exit 1;
fi

if [ $f_debug -eq 1 ]; then
        echo "Filesystem created."
fi


}

#Main
####################

while getopts "dhm:v:" o; do
    case "${o}" in
	d)
		f_debug=1
		;;
        h)
                usage
		exit 0
            ;;
        m)
		if [ "$OPTARG" = "dd" ] || [ "$OPTARG" = "rsync" ]; then
			method=$OPTARG
		else
			echo "Bad method argument choose rsync or dd" 1>&2
			exit 1
		fi
            ;;
        v)
		if [[ $OPTARG =~ ^vol-.{8,20} ]]; then 
	                volume=$OPTARG
			f_uservol=1
		else 
			echo "Volume id is not well formed" 1>&2
			exit 1
		fi
		##Check to see if the volume is valid
		 volcheck=`aws ec2 describe-volumes --volume-ids $volume`
	 	if [[ "$volcheck" =~ ^A.client.error ]]; then
			echo "Error setting the volume, please check the volume" 1>&2
			exit 1
		fi
           ;;

        *)
		usage 1>&2
                exit 1;
            ;;
    esac
done

## Shift the args back the right spot
shift $(($OPTIND - 1))

if [ -z $1 ]; then
	echo "No Directory entered" 1>&2
	exit 1
fi

if [ ! -d $1 ]; then
        echo "Argument is not a directory" 1>&2
        exit 1
fi

verifyImageAvailability
getEnvironmentVars
getAvailabilityZone
getInstance
getHostname
checkSSH
getVolume $1
attachVolume


if [ "$method" = "rsync" ]; then

	createFilesystem
	rsyncBackup $1
else
	ddBackup $1
fi


terminateInstance

echo "$volume"
exit 0