使用正则匹配进行简单的日志审计

主要通过正则匹配来检查是否存在xss，sql注入类型的攻击，因为仅使用了简单的正则匹配所以筛查效果有限，网络访问日志来源是自己搭建的服务器。除了日志审计以外，也包含了一些对linux服务器上进程的筛查

日志下载：链接：https://pan.baidu.com/s/1tCWMB4oA4juWTYjx_2qZmw
提取码：1c6l

import subprocess
import sys
from .host_info import exist
import re
import os
import requests
import time
from gevent import monkey
from gevent.pool import Pool
monkey.patch_all()
from urllib.parse import unquote
import warnings
warnings.filterwarnings("ignore")
requests.packages.urllib3.disable_warnings()


path = sys.path[0].replace("\\","/")
log_path = path+"/results/"
cpu_info = open(log_path + "/cpu_info.txt", 'a')
class netstat_Analysis:
    def __init__(self):
        self.pid=' '
        # 获取网络链接，并查看是否存在异常

    # 判断ip地址归属地，看是否是异常连接
    def check_yichangip(self):
        yichang=list()
        remot_ip=set()
        result=self.check_netstat()
        try:
            for net in result:
                # print net
                ip = net.decode("utf-8").split(' ')[1].split(':')[0]
                pid = net.decode("utf-8").split(' ')[2].split('/')[0]
                application = net.decode("utf-8").split(' ')[2].split('/')[1]
                remot_ip.add(ip)
            for i in  remot_ip:
                URL = 'http://ip-api.com/json/'+i+'?lang=zh-CN'
                try:
                    r = requests.get(URL, timeout=3,headers=headers)
                except requests.RequestException as e:
                    print (e)
                else:
                    json_data = r.json()
                    if json_data[u'status'] == 'success':
                        country= json_data[u'country'].encode('utf-8')
                        # print country
                        keyword=['中国','局域网','共享地址','本机地址','本地链路','保留地址','XX']
                        if country not  in keyword:
                            wrong_ip=i
                            info= "异常链接ip："+wrong_ip+'，ip归属地：'+country
                            file.write(info)
                            # print info
                            command1="netstat -antp  | grep "+wrong_ip+" | awk '{print $1,$5,$7}' "
                            stdin, stdout1, stderr = ssh.exec_command(command1)
                            info1= stdout1.read().splitlines()
                            # print type(info1)
                            yichang=yichang+info1
                            return yichang
                        else:
                            file.write('\n'+"查看netstat未发现国外ip地址链接"+'\n')

                    else:
                        print('查询失败,请稍后再试！')
        except:
            print("暂时未发现异常链接")
 # 对异常pid进行检测，定位文件位置
    def jiance_pid(self):
        try:
            # pid=1
            command1 = "ls -l /proc/"+self.pid
            stdout = subprocess.getoutput(command1)
            info = stdout.read().splitlines()
            mulu=info[9].decode("utf-8").split(' ')[10:]
            kezhixingwenjian=info[11].decode("utf-8").split(' ')[10:]
            yichangtixing= "pid 为"+self.pid+"的异常程序，存在位置为："+mulu[0]+"，可执行文件为"+kezhixingwenjian[0]+"，请手工进行验证。"
            cpu_info.write(yichangtixing+'\n')
            print(yichangtixing)
        except:
            print("未发现异常行为")
    # 检测cpu高于15%的程序，并获取其pid，查看是否为异常进程
    def check_cpu(self,ssh_client,n=10,):
        # 列出CPU占用率高于n的进程 默认10%
        i2,s2,e2 = ssh_client.exec_command("ps -aux | sort -nr -k 3 | awk '{if($3>=" + str(n) + ") print $2}'")
        pids = s2.read().decode()
        pids_list = pids.splitlines()
            #生成解决的命令
        pid_solve_dict = pids_list
        f = open(log_path+"/cpu_info.txt",'w')
        f1 = open(log_path+"/pid.txt",'w')
        f1.write(pids)
        f1.close()
        i,s,e = ssh_client.exec_command("ps -aux | sort -nr -k 3 | awk '{if($3>=" + str(n) + ") print }'")
        s = s.read().decode()
        if s == "":
            s = "暂未发现异常进程"
        else:
            s = "发现异常进程\n====================\n\n"+s
        f.write(s)
        f.close()
        return s,pid_solve_dict

    def portstat(self,ssh_client):
        # 查看端口状态
        if exist(log_path+"/portstat.txt"):
            return open(log_path+"/portstat.txt",'r').read()
        f = open(log_path+"/portstat.txt",'w')
        i,s,e = ssh_client.exec_command("netstat -antulp")
        s = s.read().decode()
        f.write(s)
        f.close()
        return s

    # 通过netstat查看链接，获取异常pid的程序位置
    def ps_aux(self):
        a = self.check_yichangip()
        if a!=None:
            try:
                for i in a:
                    # print i
                    self.pid = str(i).split(' ')[2].split('/')[0]
                    self.jiance_pid()
            except Exception as e:
                print(str(e))
        else:
            print("查看netstat未发现异常ip地址")


    def file_check(self,ssh_client):
        url = 'https://api.threatbook.cn/v3/file/upload';
        fields = {
            'apikey': ' fd44a5be60thisisfakec0479cd8843eda5',
            # 'sandbox_type': 'centos_7_x64',
            'sandbox_type': 'ubuntu_1704_x64',
            'run_time': 60
        }
        pid_list = set()
        file_list = {}
        file_solve_dict = {}
        for i in open(log_path + "/pid.txt", 'r').readlines():
            print("line 131 "+ i)
            f = open(log_path + "/evil_path.txt", 'w')
            pid_list.add(str(i))
            evil_pid = str(i)[0:-1]
            # 对每个pid逐个查询 ls -l /proc/$pid/cwd
            i,s,e = ssh_client.exec_command("ls -l /proc/" + evil_pid + "/cwd |awk '{print $11}'")
            s = s.read().decode()
            f.write(s)
            f.close()
            for j in open(log_path + "evil_path.txt", 'r').readlines():
                evil_path = str(j)[0:-1]
                # 筛选具有执行权限的（-perm -111）、筛选最近三天执行过的（-atime）
                i1,s1,e1 = ssh_client.exec_command("find " + evil_path + " -type f  -perm -111 -atime 0 -o -atime 1 -o -atime 2  -o -atime 3")
                s1 = s1.read().splitlines()
                for x in s1:
                    file_name = str(x)[2:-1]
                    file_solve_dict[file_name] =file_name
                    # file_list[file_name]=evil_pid
                    # 取文件名（不含路径）
                    temp_name = log_path + "/" + file_name.split('/')[-1]
                    f2 = open(temp_name, 'wb+')
                    # 把可疑文件的内容通过cat读取并写入本地的一个文件，方便上传微步
                    i2,s2,e2 = ssh_client.exec_command("cat " + file_name)
                    s2 = s2.read()
                    f2.write(s2)
                    f2.close()
                    files = {
                        'file': (temp_name, open(temp_name, 'rb'))
                    }
                    response = requests.post(url, data=fields, files=files,verify=False)
                    if response.content:
                        print(response.json())
                        if response.json()['response_code'] == 0:
                            file_list[file_name] = response.json()['data']['permalink']
                    else:
                        pass

        return file_list,file_solve_dict      

    def analysisattack(self,log):
        rules = {
            'SQLinject': {
                'UNION注入': r'union.*select.*from',
                'MSSQL报错注入': r'(and|or).*(in.*select|=convert|=concat)',
                'MySQL报错注入': r'(and|or).*(extractvalue|updatexml|floor)',
                'Oracle报错注入': r'(and|or).*(UPPER.*XMLType|utl_inaddr|CTXSYS.DRITHSX.SN|dbms_utility)',
                'MySQL盲注': r'(and|or).*(rlike|make_set|elt|sleep)',
                'Oracle盲注': r'select.*case[\s\S]when.*=.*then.*dual',
                'Oracle时间盲注': r'(and|or|begin|select).*(dbms_pipe|select.*from.*all_users|dbms_lock)',
                'MSSQL盲注': r'select.*case[\s\S]when.*=.*then',
                'MSSQL时间盲注': r'(and|or|select).*(from.*sysusers|waitfor[\s\S]delay)',
                },
            'ArbitraryFileOperation':{
                '任意文件读取/包含': r'[.]+/',
            },
            'dirtraversal':{
                'linux路径穿越': r'(etc.*passwd|etc.*shadow|etc.*hosts|.htaccess|.bash_history)',
            },
            'ArbitraryCodeExcute':{
                '任意代码执行': r'(=.*phpinfo|=.*php://|=.*\$_post\[|=.*\$_get\[|=.*\$_server\[|=.*exec\(|=.*system\(|=.*call_user_func|=.*passthru\(|=.*eval\(|=.*execute\(|=.*shell_exec\(|=.*file_get_contents\(|=.*xp_cmdshell|=.*array_map\(|=.*create_function\|=.*unserialize\(|=.*echo\()',
            },
            'struts2vuln':{
                'struts005~009': r'xwork.MethodAccessor.denyMethodExecution',
                'struts013': r'_memberAccess.*ServletActionContext',
                'struts016': r'redirect:.*context.get',
                'struts019': r'xwork2.dispatcher.HttpServletRequest',
                'struts032~057': r'@ognl.OgnlContext@DEFAULT_MEMBER_ACCESS',
            },
            'XSS':{
                '一般型': r'(<script|<img|<object|<style|<div|<table|<iframe|<meta|<body|<svg|<embed|<a|<input|<marquee|<link|<xml|<image|<html).*(alert|prompt|confirm)',
                'flashxss': r'javascript:alert',
            },
            'SSTI':{
                '一般型': r'{{.*}}',
                'Ruby模板注入': r'<%.*%>',
                'Java模板注入': r'\${.*}',
            },
            'LDAP':{
                '一般型': r'\*[\(\)|]+',
            },
            'XXE':{
                '外部实体注入': r'(<\?xml.*\?>|<!.*>|<xsl.*>)',
            }, 

        }
        f = open(log_path + '/result.txt', 'a+')
        for key, value in rules.items():
            for key2, value2 in value.items():
                try:
                    match = re.search(value2, log, re.IGNORECASE)
                    if match:
                        log = log.replace('\n', '%0a')
                        # 形式为{n},n所在的地方，会被format()方法第n个参数所代替;
                        f.write('[*]日志: {0}\n'.format(log))
                        # res = '[!]漏洞类型: {0}\t漏洞细节: {1}\t匹配规则: {2}'.format(key, key2, value2)
                        res = '[!]攻击类型: {0}\t'.format(key2)
                        # print(u'{0}'.format(res))
                        f.write('{0}\n\n'.format(res))
                        return
                except:
                    print(u'[-] 日志分析失败: {0}'.format(log))

    def analysislog(self, file, t):
        pool = Pool(t)
        logs = list()
        with open(file, 'r') as fp:
            for line in fp:
                logs.append(unquote(line.strip()))
        totalcount = len(logs)
        print(u'[*] 日志数: {0}'.format(totalcount))
        pool.map(self.analysisattack, logs)


    def create_result_txt(self,ssh_client):
        ten_day_time = "'"
        # (0,9)表示10天
        for x in range(0,9):
            day_time = time.strftime('%d/%b/%Y',time.localtime(time.time()-86400*x))
            ten_day_time = ten_day_time + day_time + "|"
        ten_day_time = ten_day_time[0:-1] + "'"
        # cat /usr/local/apache/logs/access.log | grep -E '03/Jun/2020|02/Jun/2020|01/Jun/2020|31/May/2020|30/May/2020|29/May/2020|28/May/2020|27/May/2020|26/May/2020' > access.log.bak
        
        i,s,e = ssh_client.exec_command("find / -name access.log")
        s = s.read().splitlines()
        num = 1
        for x in s:
            # 取受害机器的日志文件名,方便等会cat
            file_name = str(x)[2:-1]
            newest_log = "cat " + file_name + "| grep -E "+ ten_day_time
            i1,s1,e1 = ssh_client.exec_command(newest_log)
            f1 = open(log_path + "web_log_10_days.txt", 'a')
            # 把最近10天的日志读取到一个文件里
            s1 = s1.read().decode()
            f1.write(s1)
            f1.close()
        self.analysislog(log_path + "web_log_10_days.txt",100)





    def web_recently_analysis(self):
        headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko','Accept': 'text/html, application/xhtml+xml, image/jxr, */*','Accept-Language': 'zh-CN','Connection': 'close'}
        recent_web_attack = [[]]*10;
        
        # 取最近的10条攻击日志的ip和归属地
        command_ten_ip = "cat " + log_path + "result.txt | tail -15 |grep [*]|awk '{print $2}'"
        ten_ip = subprocess.getoutput(command_ten_ip).splitlines()
        recent_web_attack[0] = []
        recent_web_attack[1] = []
        recent_web_attack[4] = []
        for i in ten_ip:
            # 获取ip
            web_ip = str(i)
            recent_web_attack[0].append(web_ip)

            # 获取ip归属
            URL = 'http://ip-api.com/json/'+web_ip+'?lang=zh-CN'
            # print(URL)
            try:
                r = requests.get(URL, timeout=4,headers=headers)
                if r.content:
                    json_data = r.json()
                    # print(json_data)
                    if json_data[u'status'] == 'success':
                        country = json_data[u'country']
                        recent_web_attack[1].append(country)
                    else:
                        recent_web_attack[1].append("正在查询...")
                else:
                    recent_web_attack[1].append("正在查询...")
            except requests.RequestException as e:
                recent_web_attack[1].append("正在查询...")

            # 获取攻击事件条数
            command_ip_times = "cat " + log_path + "result.txt | grep '" + web_ip + "'| wc -l"
            ip_times = subprocess.getoutput(command_ip_times)
            recent_web_attack[4].append(ip_times)




        # 取最近的10条攻击日志的时间
        command_ten_time = "cat " + log_path + "result.txt | tail -15|grep [*]|awk '{print $5}'"
        ten_time = subprocess.getoutput(command_ten_time).splitlines()
        recent_web_attack[2] = []
        for i in ten_time:
            web_time = str(i)[0:-3] + ']'
            recent_web_attack[2].append(web_time)




        # 取最近的10条攻击日志的url
        command_ten_url = "cat " + log_path + "result.txt | tail -15|grep [*]|awk '{print $8}'"
        ten_url = subprocess.getoutput(command_ten_url).splitlines()
        recent_web_attack[5] = []
        for i in ten_url:
            web_url = str(i)
            print(web_url)
            recent_web_attack[5].append(web_url)
        


        # 取最近的10条攻击类型
        command_ten_type = "cat " + log_path + "result.txt | tail -15|grep [!]|awk '{print $2}'"
        ten_type = subprocess.getoutput(command_ten_type).splitlines()
        recent_web_attack[3] = []
        for i in ten_type:
            web_type = str(i)
            print(web_type)
            recent_web_attack[3].append(web_type)

        print(recent_web_attack[0])
        print(recent_web_attack[1])
        print(recent_web_attack[2])
        print(recent_web_attack[3])
        print(recent_web_attack[4])

        return recent_web_attack



    def web_all_analysis(self):
        # 最容易受到攻击的url
        url_all_dict={}
        type_all_dict={}
        command_all_url = "cat " + log_path + "result.txt|grep -a  [*]| cut -f 8 -d ' ' | sort | uniq -c | sort -n -k 1 -r |head -10"
        all_url = subprocess.getoutput(command_all_url).splitlines()
        for i in all_url:
            num1 = str(re.findall(r"(\d+) ", str(i)))[2:-2]
            attack_url = str(re.findall(r"\d (.*)", str(i)))[2:-2]
            url_all_dict[attack_url] = num1


        # 最容易受到的攻击类型
        command_all_type = "cat " + log_path + "result.txt|grep -a  [!]|grep -a -v [*]| cut -f 2 -d ' ' | sort | uniq -c | sort -n -k 1 -r |head -10"
        all_url = subprocess.getoutput(command_all_type).splitlines()
        for i in all_url:
            num2 = str(re.findall(r"(\d+) ", str(i)))[2:-2]
            print(num2)
            attack_type = str(re.findall(r"\d (.*)", str(i)))[2:-4]
            print(attack_type)
            type_all_dict[attack_type] = num2
        
        return url_all_dict,type_all_dict