package main
import (
"bufio"
"bytes"
"context"
"log"
"os"
"sync"
)
const (
logname = "log"
concurrent = 5 //并发处理数,可以根据物理内存调整
maxsize = 10 * 1024 * 1024 //每次读取的大小,可以根据物理内存调整
)
var bufs = make([][]byte, concurrent)
func main() {
var (
chs = make(map[int]chan int)
wait = new(sync.WaitGroup)
chct = make(chan int, concurrent)
ctx, cancel = context.WithCancel(context.Background())
)
File, err := os.Open(logname)
if err != nil {
log.Fatalf("Open file error,%s\n", err.Error())
}
for i := 0; i < concurrent; i++ {
chct <- i
chs[i] = make(chan int)
bufs[i] = make([]byte, maxsize)
go resolvectx(ctx, wait, i, chct, chs[i])
}
var i, n, l int
for i = range chct {
n, err = File.Read(bufs[i])
if err != nil {
wait.Wait() //等待数据全部处理完毕,然后返回
break
}
for s := 1; s < n; s++ { //如果行过长,那么效率会变低
if bufs[i][n-s] == '\n' {
n = n - s + 1
File.Seek(int64(l+n), 0)
break
}
}
l += n
wait.Add(1)
chs[i] <- n
}
cancel()
close(chct)
File.Close()
}
func resolvectx(ctx context.Context, wait *sync.WaitGroup, index int, chct, ch chan int) {
var (
err error
line []byte
length int
buf = bufio.NewReader(nil)
)
for {
select {
case <-ctx.Done():
return
case length = <-ch:
buf.Reset(bytes.NewBuffer(bufs[index][:length]))
for {
line, _, err = buf.ReadLine()
if err != nil {
break
}
_ = line
}
chct <- index
wait.Done()
}
}
}
Go1.9按行读取日志文件并处理
最新推荐文章于 2024-07-17 20:03:32 发布