AWS EC2 and S3 Bucket clean tool

This paper describe a tool which can be used to clean AWS EC2 instance/EBS and S3 bucket which are not int white list, the purpose is to save money.

---

Requirement

Install AWS CLI, https://aws.amazon.com/cli/, and then configure credential.

aws configure

Clean Up EC2 and EBS

#!/usr/bin/env ruby

require "open3"
require "json"
require "logger"

# This tool is used to cleanup AWS unknown resources.
class AWSTools
  WHITELIST_FILE = "./ec2-whitelist.data"
  LOG_FOLDER_NAME = "logs"
  LOG_FILE = "./" + LOG_FOLDER_NAME + "/main.log"
  REGIONS = {"us-east-1"=>"N.Virginia",
    "us-east-2"=>"Ohio",
    "us-west-1"=>"N.California",
    "us-west-2"=>"Oregon",
    "ap-south-1"=>"Mumbai",
    "ap-northeast-2"=>"Seoul",
    "ap-southeast-1"=>"Singapore",
    "ap-northeast-1"=>"Tokyo",
    "ap-southeast-2"=>"Sydney",
    "ca-central-1"=>"Central",
    "eu-central-1"=>"Frankfurt",
    "eu-west-1"=>"Ireland",
    "eu-west-2"=>"London",
    "eu-west-3"=>"Paris",
    "sa-east-1"=>"SaoPaulo"
  }

  def initialize
    Dir.mkdir LOG_FOLDER_NAME unless File.exists? LOG_FOLDER_NAME
    @logger = Logger.new(LOG_FILE)
    @whitelist = get_whitelist
  end

  def start
    @logger.info("Start to cleanup unknown resources.")
    begin
      REGIONS.each_key do |region|
        set_region(region)
        cleanup_instances
        cleanup_volumes
      end
    rescue => ex
      @logger.error(ex.message)
    end
    @logger.info("End to cleanup unknown resources.\n\n")
  end

  def set_region(region)
    @logger.info("Changing region: #{region}, #{REGIONS[region]}")
    ret_code, std_out, std_err = run_command("aws configure set default.region #{region}")
    if ret_code == 0
      @logger.info("Succeed to change region.")
    else
      raise "Failed to change region."
    end
  end

  def cleanup_instances
    unknown_instances = find_unknown_instances
    delete_instances(unknown_instances)
  end

  def find_unknown_instances
    unknown_instances = Array.new
    ret_code, std_out, std_err = run_command("aws ec2 describe-instances")
    if ret_code == 0
      std_out_json = JSON.parse(std_out)
      std_out_json["Reservations"].each do |reservation|
        reservation["Instances"].each do |instance|
          result = is_valid_instance(instance)
          unknown_instances.push(instance) unless result
        end
      end
      return unknown_instances
    else
      raise "Failed to get all unknown instances."
    end
  end

  def is_valid_instance(instance)
    name = find_instance_name(instance)
    return false if name.nil?
    @whitelist.each_key do |key|
      return true if name.index(key) == 0
    end
    return false
  end

  def delete_instances(instances)
    @logger.info("No unknown instances to be deleted.") if instances.size == 0
    instances.each do |instance|
      instanceId = instance['InstanceId']
      instanceName = find_instance_name(instance)
      @logger.info("Deleting instance, id: #{instanceId}, name: #{instanceName}")
      status = find_instance_status(instance)
      if status == "terminated"
        @logger.info("Instance status is already terminated.")
        next
      end
      change_instance_termination_protection(instanceId)
      terminate_instance(instanceId)
    end
  end

  def change_instance_termination_protection(instanceId)
    ret_code, std_out, std_err = run_command("aws ec2 modify-instance-attribute --instance-id #{instanceId} --disable-api-termination \"{\\\"Value\\\": false}\"")
    if ret_code == 0
      @logger.info("Succeed to change termination protection, id: #{instanceId}")
    else
      @logger.error("Failed to change termination protection, id: #{instanceId}")
    end
  end

  def terminate_instance(instanceId)
    ret_code, std_out, std_err = run_command("aws ec2 terminate-instances --instance-ids #{instanceId}")
    if ret_code == 0
      @logger.info("Succeed to delete instance, id: #{instanceId}")
    else
      @logger.error("Failed to delete instance, id: #{instanceId}")
    end
  end

  def find_instance_name(instance)
    return nil if instance["Tags"].nil?
    instance["Tags"].each do |tag|
      next if tag["Key"] != "Name"
      return tag["Value"]
    end
    return nil
  end

  def find_instance_status(instance)
    return nil if instance["State"].nil?
    return instance["State"]["Name"]
  end

  def cleanup_volumes
    unknow_volumes = find_unknown_volumes
    delete_volumes(unknow_volumes)
  end

  def find_unknown_volumes
    ret_code, std_out, std_err = run_command('aws ec2 describe-volumes --filters "Name=status,Values=available"')
    if ret_code == 0
      std_out_json = JSON.parse(std_out)
      return std_out_json["Volumes"]
    else
      raise "Failed to get all unknown volumes."
    end
  end

  def delete_volumes(volumes)
    @logger.info("No unknown volumes to be deleted.") if volumes.size == 0
    volumes.each do |volume|
      volumeId = volume['VolumeId']
      volumeName = find_volume_name(volume)
      @logger.info("Deleting volume, id: #{volumeId}, name: #{volumeName}")
      ret_code, std_out, std_err = run_command("aws ec2 delete-volume --volume-id #{volumeId}")
      if ret_code == 0
        @logger.info("Succeed to delete volume, id: #{volumeId}")
      else
        @logger.error("Failed to delete volume, id: #{volumeId}")
      end
    end
  end

  def find_volume_name(volume)
    return nil if volume["Tags"].nil?
    volume["Tags"].each do |tag|
      next if tag["Key"] != "Name"
      return tag["Value"]
    end
    return nil
  end

  def get_whitelist
    result = Hash.new
    File.open(WHITELIST_FILE, "r") do |f|
      f.each_line do |line|
        line.chomp!
        data = line.split(":")
        result[data[0]] = data[1]
      end
    end
    return result
  end

  def run_command(cmd)
    std_in, std_out, std_err, t = Open3.popen3(cmd)
    std_out_str = ""
    std_error_str = ""
    std_out.each_line do |line|
      std_out_str = std_out_str + line
    end
    std_out.close
    std_err.each_line do |line|
      std_error_str = std_error_str + line
    end
    if !std_error_str.empty?
      @logger.warn(cmd)
      @logger.warn(std_error_str)
    end
    std_err.close
    return t.value.exitstatus, std_out_str, std_error_str
  end
end

AWSTools.new.start

ec2-whitelist.data

ec2-prefix:wxhyzh2005@163.com

Clean Up S3 Buckets

#!/usr/bin/env ruby

require "open3"
require "json"
require "logger"

# This tool is used to cleanup AWS unknown resources.
class AWSTools
  WHITELIST_FILE = "./bucket-whitelist.data"
  LOG_FOLDER_NAME = "logs"
  LOG_FILE = "./" + LOG_FOLDER_NAME + "/bucket.log"
  def initialize
    Dir.mkdir LOG_FOLDER_NAME unless File.exists? LOG_FOLDER_NAME
    @logger = Logger.new(LOG_FILE)
    @whitelist = get_whitelist
  end

  def start
    @logger.info("Start to cleanup buckets.")
    begin
      cleanup_buckets
    rescue => ex
      @logger.error(ex.message)
    end
    @logger.info("End to cleanup buckets.\n\n")
  end

  def cleanup_buckets
    unknown_buckets = find_unknown_buckets
    list_buckets(unknown_buckets)
    # delete_buckets(unknown_buckets)
  end

  def list_buckets(buckets)
    buckets.each do |bucket|
      name = bucket["Name"]
      puts "#{name}"
    end
  end

  def find_unknown_buckets
    unknown_buckets = Array.new
    ret_code, std_out, std_err = run_command("aws s3api list-buckets")
    if ret_code == 0
      std_out_json = JSON.parse(std_out)
      std_out_json["Buckets"].each do |bucket|
        result = is_valid_bucket(bucket)
        unknown_buckets.push(bucket) unless result
      end
      return unknown_buckets
    else
      raise "Failed to get all buckets."
    end
  end

  def is_valid_bucket(bucket)
    name = bucket["Name"]
    return false if name.nil?
    @whitelist.each_key do |key|
      return true if name == key
    end
    return false
  end

  def delete_buckets(buckets)
    @logger.info("No unknown buckets to be deleted.") if buckets.size == 0
    buckets.each do |bucket|
      name = bucket["Name"]
      creation_date = bucket["CreationDate"]
      ret_code, std_out, std_err = run_command("aws s3 rb s3://#{name} --force")
      if ret_code == 0
        @logger.info("Succeed to delete bucket, name: #{name}, creation date: #{creation_date}")
      else
        @logger.error("Failed to delete bucket, name: #{name}, creation date: #{creation_date}")
      end
    end
  end

  def get_whitelist
    result = Hash.new
    File.open(WHITELIST_FILE, "r") do |f|
      f.each_line do |line|
        line.chomp!
        data = line.split(":")
        result[data[0]] = data[1]
      end
    end
    return result
  end

  def run_command(cmd)
    std_in, std_out, std_err, t = Open3.popen3(cmd)
    std_out_str = ""
    std_error_str = ""
    std_out.each_line do |line|
      std_out_str = std_out_str + line
    end
    std_out.close
    std_err.each_line do |line|
      std_error_str = std_error_str + line
    end
    if !std_error_str.empty?
      @logger.warn(cmd)
      @logger.warn(std_error_str)
    end
    std_err.close
    return t.value.exitstatus, std_out_str, std_error_str
  end
end

AWSTools.new.start

bucket-whitelist.data

s3-bucket-name:wxhyzh2005@163.com