go连接mysql,redis并完成日志字符处理实例

go连接mysql,redis并完成日志字符处理实例

package main

// 引入所需包
import (
    "compress/gzip"
    "fmt"
    "io/ioutil"
    "os"
    "strings"
    //"reflect"
    "github.com/garyburd/redigo/redis"
    "math"
    "strconv"
    "net/url"
    "time"
    "database/sql"
    _ "github.com/go-sql-driver/mysql"
)

//获取指定目录下的所有gz文件,包含子目录下的gz文件
func GetGzFiles(dirPth string) (files []string, err error) {
    var dirs []string
    dir, err := ioutil.ReadDir(dirPth)
    if err != nil {
        return nil, err
    }

    PthSep := string(os.PathSeparator)
    //suffix = strings.ToUpper(suffix) //忽略后缀匹配的大小写

    for _, fi := range dir {
        if fi.IsDir() { // 目录, 递归遍历
            dirs = append(dirs, dirPth+PthSep+fi.Name())
            GetGzFiles(dirPth + PthSep + fi.Name())
        } else {
            // 过滤指定格式<.gz文件>
            ok := strings.HasSuffix(fi.Name(), ".gz")
            if ok {
                files = append(files, dirPth+PthSep+fi.Name())
            }
        }
    }

    // 读取子目录下文件
    for _, dir := range dirs {
        tmpFiles, _ := GetGzFiles(dir)
        for _, file := range tmpFiles {
            files = append(files, file)
        }
    }

    return files, nil
}

/*
需要记录的值包括:
    访问时间,日期,状态码,cache返回的状态码,缓存状态码,源站返回状态码,节点IP,客户端IP,响应时间,故障原因
故障原因包括:
    (1)nginx连接cache超时(2)nginx读取cache响应头超时(3)nginx产生的其它异常(4)cache连接源站失败(5)cache缓存住的错误码(6)cache上层返回的错误码(7)cache给出的其它异常(8)其它异常
*/

type ResInfo struct {
    id    int   // 唯一ID
    timestamp int64 // 访问时间
    date      string// 日期
    dnid      int   // 域名ID

    resocde   int   // 状态码,206
    cacheCode int   // cache返回的状态码,200
    upstreamHttpStatus string   // 缓存状态码,HIT
    sourceCode int  // 源站返回状态码,200
    errorCode int   // 故障原因,0-9

    clientIp  string    // 客户端IP
    serverIp  string    // 节点IP
    reponseTime float64 // 响应时间
}

var id int = 0
func main() {
    // 更新域名列表
    db, err := sql.Open("mysql", "user:passwd@tcp(127.0.0.1:65000)/server_conf?charset=utf8")
    if err != nil {
        fmt.Printf("connect mysql failed! [%s]", err)
        return
    } else {
        fmt.Println("connect mysql ok!")
    }

    rows, err := db.Query("SELECT d_id, d_dname FROM domain")
    if err != nil {
        fmt.Printf("select failed! [%s]", err)
        return
    }

    var mapDomain map[string]int
    mapDomain = make(map[string]int)
    for rows.Next() {
        var d_id int
        var d_dname string

        rows.Columns()
        err = rows.Scan(&d_id, &d_dname)
        if err != nil {
            fmt.Printf("Get domain info failed! [%s]", err)
        }

        mapDomain[d_dname] = d_id
    }

    //for k, v := range mapDomain {
    //  fmt.Println(k, v)
    //}

    // 读取待处理的文件
    files, _ := GetGzFiles("./gzfiles")

    var ResInfoList []ResInfo
    for _, fn := range files {
        fmt.Printf("开始处理日志文件[%s]\n", fn)

        // 打开本地gz格式压缩包
        fr, err := os.Open(fn)
        if err != nil {
            panic(err)
        } else {
            println("open file success!")
        }

        // defer: 在函数退出时,执行关闭文件
        defer fr.Close()

        // 创建gzip文件读取对象
        gr, err := gzip.NewReader(fr)
        if err != nil {
            panic(err)
        }

        // defer: 在函数退出时,执行关闭gzip对象
        defer gr.Close()

        // 读取gzip对象内容
        rBuf, err := ioutil.ReadAll(gr)
        if err != nil {
            fmt.Println("[read gzip data err]: ", err)
        }

        // []uint8转string
        // fmt.Println(reflect.TypeOf(rBuf))
        strBuf := string(rBuf)
        bufList := strings.Split(strBuf, "\n")

        for _, strLine := range bufList {
            if len(strLine) == 0 {
                continue
            }

            var dnid      int
            var errorCode int
            var timestamp int64
            var resocde   int
            var cacheCode int
            var sourceCode int
            var reponseTime float64

            var date      string
            var clientIp  string
            var serverIp  string
            var upstreamHttpStatus string
            var userDefineList []string

            fragList := strings.Split(strLine, "\t")
            for i, strFrag := range fragList {
                if i == 0 {
                    strTemp := strings.Split(strFrag, "/")
                    clientIp = strTemp[0]
                    fmt.Printf("clientIp=%v\n", clientIp)
                } else if i == 1 {
                    tmp, _ := strconv.ParseFloat(strFrag, 64)
                    timestamp = int64(math.Trunc(tmp))/60*60
                    fmt.Printf("timestamp=%v\n", timestamp)

                    tm := time.Unix(timestamp, 0)
                    date = tm.Format("20060102")
                } else if i == 4 {
                    resocde, _ = strconv.Atoi(strFrag)
                    fmt.Printf("resocde=%v\n", resocde)
                } else if i == 5 {
                    upstreamHttpStatus = strFrag
                    fmt.Printf("upstreamHttpStatus=%v\n", upstreamHttpStatus)
                } else if i == 8 {
                    strTemp := strFrag
                    u, err := url.Parse(strTemp)
                    if err != nil {
                        panic(err)
                    }
                    fmt.Printf("url=%v\n", u.Host)

                    // 获取url的dnid
                    if v, ok := mapDomain[u.Host]; ok {
                        dnid = v
                    } else {
                        fmt.Printf("Domain Not Found!")
                    }
                } else if i == 13 {
                    serverIp = strFrag
                    fmt.Printf("serverIp=%v\n", serverIp)
                } else if i == 17 {
                    reponseTime, _ = strconv.ParseFloat(strFrag, 64)
                    fmt.Printf("reponseTime=%v\n", reponseTime)
                } else if i == 29 {
                    //fmt.Printf("%s \t", strFrag)
                    userDefineList = strings.Split(strFrag, "/")
                    for i, strDefine := range userDefineList {
                        if i == 5 {
                            cacheCode, _ = strconv.Atoi(strDefine)
                            //fmt.Printf("cacheCode=%v \n", cacheCode)
                            //fmt.Println(reflect.TypeOf(cacheCode))
                        }
                    }

                    //fmt.Printf("%v \t %v \n", timestamp, cacheCode)
                }
            }

            errorCode = 9

            id++
            info := ResInfo{id,timestamp,date,dnid,resocde,cacheCode,upstreamHttpStatus,sourceCode,errorCode,clientIp,serverIp,reponseTime}
            //fmt.Printf("%v\n", info)
            ResInfoList = append(ResInfoList, info)
        }
        break
    }

    // 入redis库
    // 连接redis数据库,指定数据库的IP和端口
    conn, err := redis.Dial("tcp", "127.0.0.1:65000")
    if err != nil {
        fmt.Println("Connect to redis error", err)
        return
    } else {
        fmt.Println("Connect to redis ok.")
    }

    // 函数退出时关闭连接
    defer conn.Close()
    // redis授权登陆
    conn.Do("AUTH", "xxxxxx")

    // 遍历列表并写入redis
    fmt.Printf("list len=%v\n", len(ResInfoList))
    var tm int64 = 0
    for _, info := range ResInfoList {
        //fmt.Printf("%v \n", info)
        tm = info.timestamp

        // 执行List插入
        _, err = conn.Do("LPUSH", info.timestamp, info)
        if err != nil {
            fmt.Println("redis push list failed:", err)
        } else {
            //fmt.Println("redis lpush ok!")
        }
        break
    }

    // 读取List
    infoList, err := redis.Strings(conn.Do("LRANGE", tm, 0, -1))
    if err != nil {
        fmt.Println("redis get list failed:", err)
    }

    for _, info := range infoList {
        fmt.Printf("values: %v\n", info)
    }
}