代码
- mytest.go
package main
import (
"bufio"
"fmt"
"io"
"os"
"strconv"
"strings"
"time"
"github.com/astaxie/beego/logs"
redigo "github.com/gomodule/redigo/redis"
"github.com/widuu/goini"
)
var pool *redigo.Pool
var maxmytestcount int
func Get() redigo.Conn {
return pool.Get()
}
func init_redis(cfgdbname string, conf *goini.Config) error {
redis_host := conf.GetValue(cfgdbname, "ip")
redis_port := conf.GetValue(cfgdbname, "port")
port, _ := strconv.Atoi(redis_port)
logs.Info("redis ip = ", redis_host, " ,port = ", port)
pool_size := 20
var c redigo.Conn
var err error
pool = redigo.NewPool(func() (redigo.Conn, error) {
c, err = redigo.Dial("tcp", fmt.Sprintf("%s:%d", redis_host, port))
if err != nil {
logs.Error("redis connect fail, err = ", err)
return nil, err
}
logs.Error("redis connect success")
return c, nil
}, pool_size)
return err
}
func setmyteststate(conn redigo.Conn, areaid string, numid string, state int) {
areakey := fmt.Sprintf("mytest_state_%s", areaid)
str, err := redigo.String(conn.Do("hget", areakey, numid))
if err != nil {
logs.Info("hget ", areakey, numid, ", err=", err)
myteststate := fmt.Sprintf("%d_%d", state, time.Now().Unix())
conn.Do("hset", areakey, numid, myteststate)
return
}
logs.Debug("areaid=", areaid, " ,numid=", numid, " ,state=", str)
}
func checkFileIsExist(filename string) bool {
var exist = true
if _, err := os.Stat(filename); os.IsNotExist(err) {
exist = false
}
return exist
}
func ReadLine(filePth string, ch1 chan int) error {
f, err := os.Open(filePth)
if err != nil {
return err
}
defer f.Close()
deferfunc := func(ch chan int, index *int) {
ch <- *index
close(ch)
}
bfRd := bufio.NewReader(f)
index := 0
defer deferfunc(ch1, &index)
for {
line, err := bfRd.ReadBytes('\n')
if err != nil { //遇到任何错误立即返回,并忽略 EOF 错误信息
if err == io.EOF {
return nil
}
return err
}
str := strings.Trim(string(line), "\n")
str = strings.Replace(str, "\r", "", -1)
data := strings.Split(str, "|")
if len(data) != 2 {
logs.Error("data len = ", len(data))
continue
} else {
user := data[0]
userdata := strings.Split(user, "_")
if len(userdata) != 2 {
continue
}
areaid := userdata[0]
userid := userdata[1]
numidstr := data[1]
numid := strings.Split(numidstr, ",")
dealWriteRedis(areaid, userid, numid)
}
index++
}
return err
}
func dealWriteRedis(areaid string, userid string, numids []string) {
redis_conn := Get()
defer redis_conn.Close()
setmyteststate(redis_conn, areaid, userid, 1)
key := fmt.Sprintf("mytest_list_%s_%s", areaid, userid)
count, err := redigo.Int64(redis_conn.Do("ZCOUNT", key, 0, 4294967295))
if err != nil {
logs.Error("zcount err=", err, " ,key = ", key)
return
} else {
logs.Debug("key = ", key, " , mytest count = ", count)
}
for _, numid := range numids {
// redis_conn.Do("zadd", key, 1582992000, numid)
redis_conn.Send("ZADD", key, 1582992000, numid) // 2020-03-01 --> 1582992000
}
redis_conn.Flush()
for i := 0; i < len(numids); i++ {
redis_conn.Receive()
}
if int64(len(numids))+int64(count) > int64(maxmytestcount) {
fc, _ := redigo.Int64(redis_conn.Do("ZCOUNT", key, 0, 4294967295))
if fc > int64(maxmytestcount) {
logs.Info("fc=", fc, " fclimit=", maxmytestcount, " ,key=", key)
redis_conn.Do("ZREMRANGEBYRANK", key, 0, fc-int64(maxmytestcount)-1)
}
}
}
func main() {
logs.SetLogger(logs.AdapterFile, `{"filename":"mytest.log","level":7,"maxlines":0,"maxsize":0,"daily":false,"maxdays":1000,"color":true}`)
conf := goini.SetConfig("./config.ini")
erredis := init_redis("redis_db", conf)
if erredis != nil {
logs.Error("redis connect fail, err = ", erredis)
return
}
init_maxfcount := conf.GetValue("config", "maxmytestcount")
maxmytestcount_t, err4 := strconv.Atoi(init_maxfcount)
maxmytestcount = maxmytestcount_t
if err4 != nil {
logs.Error("config, maxmytestcount = ", maxmytestcount, " , err = ", err4)
return
}
mytestfile := conf.GetValue("config", "input_filename")
if mytestfile == "" {
logs.Error("config, mytestfile = ", mytestfile)
return
}
logs.Info("mytestfile = ", mytestfile, " ,maxmytestcount=", maxmytestcount)
ch1 := make(chan int)
if checkFileIsExist(mytestfile) { //如果文件存在
go ReadLine(mytestfile, ch1)
}
for i := range ch1 {
logs.Info("deal end,line = ", i)
}
}
配置文件
- config.ini
[redis_db]
ip = 127.0.0.1
port = 6379
[config]
maxmytestcount = 150
input_filename = datainput.txt
编译
- windows环境下可以编译linux运行程序,但不能直接运行linux版本运行程序。
windows版本
go mod init mytest
go build mytest.go
linux版本
set CGO_ENABLED=0
set GOOS=linux
set GOARCH=amd64
go mod init mytest
go build mytest.go
运行
windows版本
mytest.exe
linux版本(linux环境下运行)
./mytest
其他
- go redis 示例中使用了 管线技术(Send,Flush,Receive)。
- 需要使用Go 1.13以上版本。设置代理配置命令:
go env -w GOSUMDB="sum.golang.google.cn" # 更换为国内的校验源,默认sum.golang.org
go env -w GOPROXY="https://goproxy.io,direct" # 解决代理
go env -w GO111MODULE="on"
go env -w GOPRIVATE=*.gitlab.com,*.gitee.com # 代理跳过私有库设置