欢迎使用CSDN-markdown编辑器

data_op.py
#!/usr/bin/env python
#_*_ coding:utf-8 _*_

'''
Created on 2017��8��23��

@author: ZHANGHENG266
'''

import MySQLdb
import time

class MysqldbHelper(object):

    def __init__(self, host, user, passwd,db):
        self.Host = host
        self.User = user
        self.Passwd = passwd
        self.DB = db
        self.Port = 3306
        self.Charset = 'utf8'    

    # 获取数据库连接
    def getCon(self):
        try:
            if self.DB not in self.check_db():
                print "数据库不存在。。准备创建数据库"
                self.create_db()
                conn = MySQLdb.Connect(host=self.Host, 
                                       user=self.User, 
                                       passwd=self.Passwd, 
                                       db=self.DB, 
                                       port=self.Port,
                                       charset=self.Charset)
            else:
                conn = MySQLdb.Connect(host=self.Host, 
                                       user=self.User, 
                                       passwd=self.Passwd, 
                                       db=self.DB, 
                                       port=self.Port,
                                       charset=self.Charset)               
            return conn
        except MySQLdb.Error, e:
            print "Mysqldb Error:%s" % e

    #列出所有的数据库        
    def check_db(self):
        try:
            db_list = []
            conn = MySQLdb.Connect(host=self.Host, 
                                   user=self.User, 
                                   passwd=self.Passwd, 
                                   port=self.Port,
                                   charset=self.Charset)
            cur = conn.cursor()
            cur.execute("show databases")
            for db in cur.fetchall():
                db_list.append(db[0])
            return db_list
        except MySQLdb.Error, e:
            print "Mysqldb Error:%s" % e
        finally:
            cur.close()
            conn.close()

    #判断并创建数据库
    def create_db(self):
        try:
            conn = MySQLdb.Connect(host=self.Host, 
                                   user=self.User, 
                                   passwd=self.Passwd, 
                                   port=self.Port,
                                   charset=self.Charset)
            cur = conn.cursor()
            while True:
                new_db_name = raw_input("请输入要创建的数据库名： ").strip()
                if len(new_db_name) == 0:continue
                if new_db_name not in self.check_db():
                    print "开始建表..........."
                    cur.execute("create database %s" %new_db_name)
                    if new_db_name in self.check_db():
                        print u"数据库%s创建成功" %new_db_name
                        return new_db_name
                        break                    
                else:
                    print "数据库 %s 已经存在，请重新输入" %new_db_name
                    time.sleep(1)
        except MySQLdb.Error, e:
            print "Mysqldb Error:%s" % e
        finally:
            cur.close()
            conn.close()

    #删除数据库
    def drop_db(self):
        try:
            conn = MySQLdb.Connect(host=self.Host, 
                                   user=self.User, 
                                   passwd=self.Passwd, 
                                   port=self.Port,
                                   charset=self.Charset)
            cur = conn.cursor()
            while True:
                db_name = raw_input("请输入要删除的数据库名： ").strip()
                if len(db_name) == 0:continue
                if db_name in self.check_db():
                    print "开始删除..........."
                    cur.execute("drop database %s" %db_name)
                    if db_name not in self.check_db():
                        print u"数据库%s删除成功" %db_name
                        return db_name
                        break                    
                else:
                    print "数据库 %s 不存在，请重新输入" %db_name
                    time.sleep(1)
        except MySQLdb.Error, e:
            print "Mysqldb Error:%s" % e
        finally:
            cur.close()
            conn.close()

    #获取当前数据库里所有标的列表
    def check_table(self):
        try:
            tb_list = []
            con = self.getCon()
            cur = con.cursor()
            cur.execute("select database()")
            print "当前所在数据库： %s" %cur.fetchall()[0]
            cur.execute("show tables")
            for tb in cur.fetchall():
                tb_list.append(tb[0])
            return tb_list
        except MySQLdb.Error, e:
            print "Mysqldb Error:%s" % e
        finally:
            cur.close()
            con.close()

    #判断并创建表
    def create_table(self,table_structure):
        try:
            con = self.getCon()
            cur = con.cursor()
            while True:
                input_table_name = raw_input("请输入要创建的表名： ").strip()
                if len(input_table_name) == 0:continue
                if input_table_name not in self.check_table():
                    print "开始建表..........."
                    cur.execute("create table %s%s;" %(input_table_name,table_structure))                    
                    time.sleep(1)
                    self.succ_table(input_table_name)
                    return input_table_name
                else:
                    print "表 %s 已经存在，请重新输入" %input_table_name
                    time.sleep(1)
        except MySQLdb.Error, e:
            print "Mysqldb Error:%s" % e
        finally:
            cur.close()
            con.close()

    #如果能查看到表结构，说明建表成功
    def succ_table(self, cr_table):
        try:
            con = self.getCon()
            cur = con.cursor()
            cur.execute("describe %s" %cr_table)
            print "\tField \t\tType \t\tNull \t\tKey \t\tDefault \t\tExtra"
            print "\t-----------------------------------------------------\n"
            for i in cur.fetchall():
                print "\t %s" %str(i[0])
            print "\n表 %s 创建成功" %cr_table
        except MySQLdb.Error, e:
            print "Mysqldb Error:%s" % e
        finally:
            cur.close()
            con.close()

    #判断并删除表
    def drop_table(self):
        try:
            con = self.getCon()
            cur = con.cursor()
            while True:
                drop_table_name = raw_input("请输入要删除的表名： ").strip()
                if len(drop_table_name) == 0:continue
                if drop_table_name in self.check_table():
                    print "开始删除..........."
                    cur.execute("drop table %s" %drop_table_name)
                    print "已经删除..........."
                    break
                else:
                    print "表 %s 不存在，请重新输入" %drop_table_name
                    time.sleep(1)
        except MySQLdb.Error, e:
            print "Mysqldb Error:%s" % e
        finally:
            cur.close()
            con.close()

    # 查询方法，使用con.cursor(MySQLdb.cursors.DictCursor),返回结果为字典
    def select(self, sql):
        try:
            con = self.getCon()
            cur = con.cursor()
            cur.execute(sql)
            data = cur.fetchall()
            return data
        except MySQLdb.Error, e:
            print "Mysqldb Error:%s" % e
        finally:
            cur.close()
            con.close()

    #修改方法（增加、删除、修改）
    def update(self, sql, params):
        try:
            con = self.getCon()
            cur = con.cursor()
            cur.execute(sql, params)
            con.commit()
        except MySQLdb.Error, e:
            print "Mysqldb Error:%s" % e
        finally:
            cur.close()
            con.close()


if __name__ == "__main__":
    dbname=MysqldbHelper('192.168.72.241','root','redhat','kwlist')
    sql = "select * from kwlist_tb"
    fc = dbname.select(sql)
    print "数据库信息如下："
    print fc

    list_q =dbname.check_table()
    print list_q

main.py
#!/usr/bin/env python
#_*_ coding:utf-8 _*_

'''
Created on 2017年8月24日

@author: ZHANGHENG266
'''

import sys
import time
from Data_op import MysqldbHelper
from Unzip import eachFile

# 引导精灵
def program_guidespeak(message, with_guid=True):
    if with_guid:
        print 'manbao: %s' % message
    else:
        print message

# 程序说明
def program_showbanner():
    print u'''程序说明:
        专业公司上传日志，讲日志文件归档，并分析查看日志告警。。。'''

# 画矩形盒
def program_drawbox(box_width, menu_list):
    side_sta_width = 5
#    side_end_width = side_sta_width + box_width
    print ''.join([' '*(side_sta_width-1), '-'*box_width])
    for cur_index, cur_item in enumerate(menu_list,start=0):
        cur_menu_item = '%s. %s' % (cur_index, cur_item)
        print '|'.join([' '*(side_sta_width-1),
                        cur_menu_item.ljust(box_width-side_sta_width, ' ')]),
        print '|'
    print ''.join([' '*(side_sta_width-1), '-'*box_width])

# 首页选择
def login_select(menu_list):
    while True:
        print "欢迎来到主菜单！"
        program_drawbox(75 ,menu_list)
        login_choice = raw_input('please input your choice: ')
        if login_choice not in map(str, range(len(menu_list))):
            print 'errors: valid login choice number. try again.'
            continue
        if login_choice == '0':
            # 日志
            print "欢迎来到日志初处理界面"
            log_select(Log_menu)
            continue
        elif login_choice == '1':
            # 数据库
            print "欢迎使用数据库"
            Database_select(Database_menu)
            continue     
        elif login_choice == '2':
            # 表单
            print "欢迎创建表单"
            Sheet_select(Sheet_menu)
            continue
        elif login_choice == '3':
            # 分析日志
            print "欢迎分析日志"
            Log_anlysis(log_anlysis_menu)
            continue
        else:
            # 退出
            sys.exit('程序已退出')

# 日志选择
def log_select(menu_list):
    while True:
        program_drawbox(75 ,menu_list)
        select_choice = raw_input('please input your choice: ')
        if select_choice not in map(str, range(len(menu_list))):
            print 'errors: valid login choice number. try again.'
            continue
        #解压日志
        if select_choice == '0':
            eachFile("D:\Test")
            continue
        #日志归档
        elif select_choice == '1':
            continue
        #返回主菜单
        elif select_choice == '2':
            return False

def Database_select(menu_list):
    dbname=MysqldbHelper('192.168.72.241','root','redhat','wanjiayiliao')
    while True:
        program_drawbox(75 ,menu_list)
        select_choice = raw_input('please input your choice: ')
        if select_choice not in map(str, range(len(menu_list))):
            print 'errors: valid login choice number. try again.'
            continue
        #创建数据库
        if select_choice == '0':
            dbname.create_db()
            continue
        #查看数据库清单
        elif select_choice == '1':
            print dbname.check_db()
            continue
        #删除数据库
        elif select_choice == '2':
            dbname.drop_db() 
            continue
        #返回主菜单
        elif select_choice == '3':
            return False

def Sheet_select(menu_list):
    dbname=MysqldbHelper('192.168.72.241','root','redhat','wanjiayiliao')
    table_structure = "(ID INT NOT NULL ,\
    AppName VARCHAR(255) NULL,\
    Submission_date DATE,\
    Log_uses VARCHAR(255) NULL,\
    LogName VARCHAR(255) NULL,\
    KWLevel1 VARCHAR(255) NULL,\
    KWLevel2 VARCHAR(255) NULL,\
    Log_archive LONGTEXT NULL)"
    while True:
        program_drawbox(75 ,menu_list)
        select_choice = raw_input('please input your choice: ')
        if select_choice not in map(str, range(len(menu_list))):
            print 'errors: valid login choice number. try again.'
            continue
        #查看当前数据库中的表清单
        if select_choice == '0':
            print dbname.check_table()
            continue
        #创建表单
        elif select_choice == '1':
            dbname.create_table(table_structure)
            continue
        #查看表结构
        elif select_choice == '2':
            table_name = raw_input("请输入表名称： ").strip()
            dbname.succ_table(table_name)
            continue 
        #查询表内容
        elif select_choice == '3':
            sql = "select * from test4"
            print dbname.select(sql)
            pass
            continue
        #更新表信息
        elif select_choice == '4':
            sql = "insert into test4 values(%s,%s,%s,%s,%s,%s,%s,%s)"
            param = ("2",'11', '2017-01-01','lala','lala','lala','lala','lala')
            dbname.update(sql, param)
            pass
            continue
        #删除某张表
        elif select_choice == '5':
            dbname.drop_table()
            continue
        #主菜单
        elif select_choice == '6':
            return False

def Log_anlysis(menu_list):
    while True:
        program_drawbox(75 ,menu_list)
        select_choice = raw_input('please input your choice: ')
        if select_choice not in map(str, range(len(menu_list))):
            print 'errors: valid login choice number. try again.'
            continue
        #将日志入库
        if select_choice == '0':
            pass
            continue
        #关键字统计
        elif select_choice == '1':
            pass
            continue
        #返回主菜单
        elif select_choice == '2':
            return False

def main():
    program_showbanner()
    program_guidespeak(u'''免责声明:
    欢迎到日志分析平台,感谢使用!''', False)
    program_guidespeak(u'''正在载入....''', False)
    time.sleep(2)
    # 登陆系统
    login_select(login_menu)

if __name__ == '__main__':
    login_menu = [u'日志初处理', u'数据库模块', u'表单的模块', u'分析日志块', u'退出主程序']
    Log_menu = [u'解压日志文件',u'原始日志归档',u'返回主菜单块']
    Database_menu = [u'创建数据库', u'数据库清单', u'删除数据库',u'返回主菜单']
    Sheet_menu = [u'查看表清单', u'创建新的表', u'查看表结构', u'查询表内容',u'更新表信息',u'删除某张表',u'返回主菜单']
    log_anlysis_menu = [u'将日志入库', u'关键字统计',u'返回主菜单']
    main()









#    eachFile("D:\Test")                     #讲需要处理的日志文件解压

#    print dbname.check_db()                      #查看所有数据库    
#    print dbname.check_table()                   #查看数据库中的所有表
#    
#    dbname.succ_table("test3")

#    dbname.create_table(table_structure)        #创建表
#    dbname.drop_table()                         #删除表

#    sql = "select * from test4"        #查询表信息
#    fc = dbname.select(sql)
#    print "数据库信息如下："
#    print fc
#    
#    #增加表信息(还有问题，待分析)
#    sql = "insert into test4 values(%s,%s,%s,%s,%s,%s,%s,%s)"
#    param = ("2",'11', '2017-01-01','lala','lala','lala','lala','lala')
#    dbname.update(sql, param)

#    print "main"
#    dbname=MysqldbHelper('192.168.72.241','root','redhat','wanjiayiliao')
##    dbname=MysqldbHelper('192.168.72.241','root','redhat','kwlist')
#    table_structure = "(ID INT NOT NULL ,\
#    AppName VARCHAR(255) NULL,\
#    Submission_date DATE,\
#    Log_uses VARCHAR(255) NULL,\
#    LogName VARCHAR(255) NULL,\
#    KWLevel1 VARCHAR(255) NULL,\
#    KWLevel2 VARCHAR(255) NULL,\
#    Log_archive LONGTEXT NULL)"
#    main()    

unzip.py
#!/usr/bin/env python
#_*_ coding:utf-8 _*_

'''
Created on 2017年8月22日

@author: ZHANGHENG266
'''

import os
import zipfile

#解决办法1（有问题）        
#zip与tar类似，先读取多个文件名称，然后解压。
#def un_zip(file_name,filepath):
#    try: 
#        """unzip zip file"""
#        zip_file = zipfile.ZipFile(file_name)
#        if os.path.isdir(filepath):
#            pass
#        else:
#            os.mkdir(filepath)
#        for names in zip_file.namelist():
#            zip_file.extract(names,filepath)
#        zip_file.close()
#    except WindowsError, e:
#        un_zip(file_name,filepath)
#        print "Mysqldb Error:%s" % e


#解决办法2  
def un_zip(file_name):
    """unzip zip file"""
    zip_file = zipfile.ZipFile(file_name)
    if os.path.isdir(file_name.replace(".zip","")):
        pass
    else:
        os.mkdir(file_name.replace(".zip",""))
    for names in zip_file.namelist():
        zip_file.extract(names,file_name.replace(".zip",""))
    zip_file.close()

#遍历文件夹，并将文件解压到当前目录
def eachFile(filepath):
    pathDir = os.listdir(filepath)      #获取当前路径下的文件名，返回List
    print pathDir     
    for s in pathDir:
        newDir=os.path.join(filepath,s)     #将文件命加入到当前文件路径后面
        print newDir
        print s
        if os.path.isfile(newDir):         #如果是文件
            if ".zip" in s :
                print os.path.dirname(newDir)
                un_zip(newDir) 
#                os.path.dirname(newDir)              
                c= set(os.listdir(filepath)) - set(pathDir)
                for i in c:
                    eachFile(os.path.join(filepath,i))    
            else:
                break
        else:
            eachFile(newDir)                #如果不是文件，递归这个文件夹的路径   

if __name__ == '__main__':
    eachFile("D:\Test")
    pass

#!/usr/bin/env python
#_*_ coding:utf-8 _*_

'''
Created on 2017年8月28日

@author: ZHANGHENG266
'''
import os
import re

#遍历目录，使生成单行日志

def OneLine(filename):
    fi = open (filename,'r')
    mline = 0
    for line in open (filename,'r'):
        linefi = fi.readline()
        print linefi
        result = re.findall(r"\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b", linefi[0:16])
        print result
        for kw in kwcr:
            print "lalal"
            if kw == "[ERROR] [":
                fo = open ("D:\log\error.txt",'a')
                print "error"
                if linefi[0:len(kw)] != kw:
                    fo.write(linefi.strip() + "<CR>")
                    mline=1
                else:
                    if mline==1:
                        fo.write("\n" + linefi[:-1])
                        mline=0
                    else:
                        fo.write(linefi.strip()+"<CR>")
            elif kw == "[INFO ] [":
                fo = open ("D:\log\info.txt",'a')
                print "info"
                if linefi[0:len(kw)] != kw:
                    fo.write(linefi.strip() + "<CR>")
                    mline=1
                else:
                    if mline==1:
                        fo.write("\n" + linefi[:-1])
                        mline=0
                    else:
                        fo.write(linefi.strip()+"<CR>")
            elif kw == "[WARN ]":
                fo = open ("D:\log\warn.txt",'a')
                print "warn"
                if linefi[0:len(kw)] != kw:
                    fo.write(linefi.strip() + "<CR>")
                    mline=1
                else:
                    if mline==1:
                        fo.write("\n" + linefi[:-1])
                        mline=0
                    else:
                        fo.write(linefi.strip()+"<CR>")
#            elif kw == "IP":
#                fo = open ("D:\log\localhost.txt",'a')
#                ip = result = re.findall(r"\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b", linefi[0:16])
#                print "access"
#                if ip:
#                    fo.write(linefi.strip() + "<CR>")
#                    mline=1
#                else:
#                    if mline==1:
#                        fo.write("\n" + linefi[:-1])
#                        mline=0
#                    else:
#                        fo.write(linefi.strip()+"<CR>")

    fi.close()
    fo.close()

def eachFile(filepath):
    pathDir = os.listdir(filepath)
    for s in pathDir:
        newDir=os.path.join(filepath,s)
        if os.path.isfile(newDir) :
            if os.path.splitext(newDir)[1]==".txt" or os.path.splitext(newDir)[1]==".log":
                print newDir
                OneLine(newDir)  
            else:
#                kwcount(newDir,kw)
                pass
        else:
            eachFile(newDir)


if __name__ == '__main__':
    print "lalal"
    kwcr = ["[ERROR] [","[INFO ] [","[WARN ]"]
    eachFile("D:\Test")
    pass

#-*- coding: UTF-8 -*-  
import os
import zipfile
import gzip
import tarfile

if os.path.exists("D:/test"):
    message='文件夹存在'
else:
    os.mkdir("D:/test")
    message='需要新建文件夹'
print message

#解压zip
def un_zip(file_name):
    """unzip zip file"""
    zip_file = zipfile.ZipFile(file_name)
    f_name = file_name.replace(".zip","")
#    print f_name
    if os.path.isdir(f_name):
        pass
    else:
        os.mkdir(f_name)
    for names in zip_file.namelist():
        zip_file.extract(names, f_name + "/")
    zip_file.close()  

#解压tar
def un_tar(file_name):
    tar = tarfile.open(file_name)
    names = tar.getnames()
    f_name = file_name.replace(".tar","")
    if os.path.isdir(f_name):
        pass
    else:
        os.mkdir(f_name)
    #因为解压后是很多文件，预先建立同名目录
    for name in names:
        tar.extract(name, f_name + "/")
    tar.close()

def un_gz(file_name):
    """ungz zip file"""
    f_name = file_name.replace(".gz", "")
    #获取文件的名称，去掉
    g_file = gzip.GzipFile(file_name)
    #创建gzip对象
    open(f_name, "w+").write(g_file.read())
    #gzip对象用read()打开后，写入open()建立的文件里。
    g_file.close()
    #关闭gzip对象 



def dirlist(path,allfile):
    filelist = os.listdir(path)
    for filename in filelist:
        filepath = os.path.join(path,filename)
        if os.path.isdir(filepath):
            dirlist(filepath,allfile) 
        else:
            if tarfile.is_tarfile(filepath):
                un_tar(filepath) 
            if zipfile.is_zipfile(filepath):
                un_zip(filepath) 
    return allfile