源码目录
packetbeat
|--------sniffer//报文捕获或读取报文文件
|--------decoder//报文数据处理
|--------flowID //流管理
|--------protos //协议处理如TCP、UDP、MYSQL
|
|--------tcp
|--------udp
|--------redis
|--------mysql
|--------drda
启动
Packbeats入口结构:
type packetbeat struct {
//配置
config config.Config
cmdLineArgs flags
pub *publish.PacketbeatPublisher
//报文捕获
sniff *sniffer.SnifferSetup
//服务开关
services []interface {
Start()
Stop()
}
}
//报文捕获的结构包含报文文件,pcap抓包等方式
type SnifferSetup struct {
pcapHandle *pcap.Handle
afpacketHandle *afpacketHandle
pfringHandle *pfringHandle
config *config.InterfacesConfig
isAlive bool
dumper *pcap.Dumper
// bpf filter
filter string
// Decoder *decoder.DecoderStruct
worker Worker
DataSource gopacket.PacketDataSource
}
packetbeat的init函数中调用setupSniffer
// init packetbeat components
func (pb *packetbeat) init(b *beat.Beat) error {
…………
err = pb.setupSniffer()
if err != nil {
return fmt.Errorf("Initializing sniffer failed: %v", err)
}
return nil
}
setupSniffer函数中调用sniffer模块,并将createWorker作为参数传递给sniff的初始化接口
func (pb *packetbeat) setupSniffer() error {
config := &pb.config
…………
pb.sniff = &sniffer.SnifferSetup{}
return pb.sniff.Init(false, filter, pb.createWorker, &config.Interfaces)
}
createWorker主要是创建一个decoder模块,将返回的worker赋值给SnifferSetup.work,这样在sniffer模块中就能调用decoder中的报文处理函数了,这个函数就是work接口的OnPacket
func (pb *packetbeat) createWorker(dl layers.LinkType) (sniffer.Worker, error) {
……………………
worker, err := decoder.New(f, dl, icmp4, icmp6, tcp, udp)
if err != nil {
return nil, err
}
if f != nil {
pb.services = append(pb.services, f)
}
return worker, nil
}
初始化完成后,在packetbeat的run中调用sniff的run,这样sniff报文捕获模块就运行起来了
func (pb *packetbeat) Run(b *beat.Beat) error {
…………
go func() {
defer wg.Done()
err := pb.sniff.Run()
if err != nil {
errC <- fmt.Errorf("Sniffer main loop failed: %v", err)
}
}()
…………
}
报文处理
报文捕获模块,获取报文数据
func (sniffer *SnifferSetup) Run() error {
………………
//这里的data就是获取到的报文数据
data, ci, err := sniffer.DataSource.ReadPacketData()
………………
sniffer.worker.OnPacket(data, &ci)
………………
}
Decoder
sniffer中的sniffer.worker.OnPacket(data, &ci)调用的就是这里的OnPacket
func (d *Decoder) OnPacket(data []byte, ci *gopacket.CaptureInfo) {
…………
for len(data) > 0 {
…………
//协议处理
processed, err = d.process(&packet, currentType)
…………
}
…………
}
func (d *Decoder) process(
packet *protos.Packet,
layerType gopacket.LayerType,
) (bool, error) {
…………
//根据协议类型对应处理相应的协议,这里是TCP协议
case layers.LayerTypeTCP:
debugf("TCP packet")
d.onTCP(packet)
return true, nil
…………
}
func (d *Decoder) onTCP(packet *protos.Packet) {
src := uint16(d.tcp.SrcPort)
dst := uint16(d.tcp.DstPort)
id := d.flowID
if id != nil {
//添加新流
id.AddTCP(src, dst)
}
packet.Tuple.SrcPort = src
packet.Tuple.DstPort = dst
packet.Payload = d.tcp.Payload
if id == nil && len(packet.Payload) == 0 && !d.tcp.FIN {
// We have no use for this atm.
debugf("Ignore empty non-FIN packet")
return
}
packet.Tuple.ComputeHashebles()
//处理会话层协议
d.tcpProc.Process(id, &d.tcp, packet)
}
func (tcp *TCP) Process(id *flows.FlowID, tcphdr *layers.TCP, pkt *protos.Packet) {
…………
case seqGT:
// lastSeq > tcpStartSeq => overlapping TCP segment detected. shrink packet
delta := lastSeq - tcpStartSeq
if isDebug {
debugf("Overlapping tcp segment. last_seq %d, seq: %d, delta: %d",
lastSeq, tcpStartSeq, delta)
}
pkt.Payload = pkt.Payload[delta:]
tcphdr.Seq += delta
}
}
stream.addPacket(pkt, tcphdr)
}
func (stream *TCPStream) addPacket(pkt *protos.Packet, tcphdr *layers.TCP) {
conn := stream.conn
mod := conn.tcp.protocols.GetTCP(conn.protocol)
…………
//协议入口
if len(pkt.Payload) > 0 {
conn.data = mod.Parse(pkt, &conn.tcptuple, stream.dir, conn.data)
}
//收到FIN
if tcphdr.FIN {
conn.data = mod.ReceivedFin(&conn.tcptuple, stream.dir, conn.data)
}
}
协议注册
// list of protocol plugins
type ProtocolsStruct struct {
all map[Protocol]Plugin
tcp map[Protocol]TCPPlugin
udp map[Protocol]UDPPlugin
}
//协议集合
注册协议
func (s ProtocolsStruct) Init(
testMode bool,
results publish.Transactions,
configs map[string]*common.Config,
) error {
for proto := range protocolSyms {
logp.Info("registered protocol plugin: %v", proto)
}
for name, config := range configs {
// XXX: icmp is special, ignore here :/
if name == "icmp" {
continue
}
proto, exists := protocolSyms[name]
if !exists {
logp.Err("Unknown protocol plugin: %v", name)
continue
}
plugin, exists := protocolPlugins[proto]
if !exists {
logp.Err("Protocol plugin '%v' not registered (%v).", name, proto.String())
continue
}
if !config.Enabled() {
logp.Info("Protocol plugin '%v' disabled by config", name)
continue
}
inst, err := plugin(testMode, results, config)
if err != nil {
logp.Err("Failed to register protocol plugin: %v", err)
return err
}
s.register(proto, inst)
}
return nil
}
数据包入口函数
//entry point
func (drda *drdaPlugin) Parse(pkt *protos.Packet, tcptuple *common.TCPTuple,
dir uint8, private protos.ProtocolData) protos.ProtocolData
协议
以mysql举例,mysql包的初始化函数init调用proto包中的register
func init() {
protos.Register("mysql", New)
}
func Register(name string, plugin ProtocolPlugin) {
proto := Protocol(len(protocolNames))
if p, exists := protocolSyms[name]; exists {
// keep symbol table entries if plugin gets overwritten
proto = p
} else {
protocolNames = append(protocolNames, name)
protocolSyms[name] = proto
}
protocolPlugins[proto] = plugin
}
注册的协议保存到全局变量中
var (
protocolPlugins = map[Protocol]ProtocolPlugin{}
protocolSyms = map[string]Protocol{}
)
Output
协议调用发送接口
mysql.results.PublishTransaction(event)
func (p *PacketbeatPublisher) PublishTransaction(event common.MapStr) bool {
select {
case p.trans <- event:
return true
default:
// drop event if queue is full
return false
}
}
func (p *PacketbeatPublisher) Start() {
p.wg.Add(1)
go func() {
defer p.wg.Done()
for {
select {
case <-p.done:
return
case event := <-p.trans:
p.onTransaction(event)
}
}
}()
………………
}
func (p *PacketbeatPublisher) onTransaction(event common.MapStr) {
…………
p.client.PublishEvent(event)
}
//kafka
func (k *kafka) PublishEvent(
signal op.Signaler,
opts outputs.Options,
data outputs.Data,
) error {
mode, err := k.getMode(opts)
if err != nil {
return err
}
return mode.PublishEvent(signal, opts, data)
}
配置文件解析
//配置结构
type BeatConfig struct {
Shipper publisher.ShipperConfig `config:",inline"`
Output map[string]*common.Config `config:"output"`
Logging logp.Logging `config:"logging"`
Processors processors.PluginConfig `config:"processors"`
Path paths.Path `config:"path"`
}
//加载配置文件
func LoadFile(path string)
//配置文件是yaml格式
调用第三方yaml库解析
yaml.Unmarshal
//
cfg.Unpack