一个稍微复杂的awk & sed应用

最近做了一个文本 CDR 转换器，从 A 样式转换为 B 样式，如下所示。

 

A 样式

RECORD #input_id 1282892739x001_0011058 #output_id #input_type GGSNPDPRecord #output_type GGSNPDPRecord #addkey #source_id ggsn01 #filename ggsn01_20100827121321_13251 F recordType 13 F servedIMSI 454063302785690 F ggsniPBinV4Address 203078047018 F chargingID 143CFE18 F sgsniPBinV4Address 203126001225 F accessPointName 11736D6172746F6E652D766F6461666F6E65 F pdpType 0121 F servediPBinV4Address 010052130231 F dynamicAddressFlag FF F dataVolumeGPRSUplink 76F4 F dataVolumeGPRSDownlink 09B605 F changeCondition 02 F changeTime 1008271213222B0800 F recordOpeningTime 1008271212382B0800 F duration 2C F causeForRecClosing 00 F recordSequenceNumber 03 F nodeID 6767736E3031 F identifier 04007F00050202000000010001 F changeTime 1008271213222B0800 F serviceIdentifier 54 F sivolumeUplink 711C F sivolumeDownlink 09B2C5 F serviceIdentifier 62 F sivolumeUplink 05D8 F sivolumeDownlink 0340 F serviceClass 62 F volumeUplink 05D8 F volumeDownlink 0340 F serviceClass 4B F volumeUplink 711C F volumeDownlink 09B2C5 F apnSelectionMode 00 F servedMSISDN 198612398426688 F chargingCharacteristics 0800 F chChSelectionMode 00 F sgsnPLMNIdentifier 525F01 F servedIMEI 0121610002016205 F rATType 01 F localSequenceNumber 0DDEF73B .

 

B 样式

 

Record (1) "GGSNPDPRecord" "recordType" = "13" "servedIMSI" = "454063302785690" "ggsniPBinV4Address" = "203078047018" "chargingID" = "143CFE18" "sgsniPBinV4Address" = "203126001225" "accessPointName" = "11736D6172746F6E652D766F6461666F6E65" "pdpType" = "0121" "servediPBinV4Address" = "010052130231" "dynamicAddressFlag" = "FF" "dataVolumeGPRSUplink" = "76F4" "dataVolumeGPRSDownlink" = "09B605" "changeCondition" = "02" "changeTime" = "1008271213222B0800" "recordOpeningTime" = "1008271212382B0800" "duration" = "2C" "causeForRecClosing" = "00" "recordSequenceNumber" = "03" "nodeID" = "6767736E3031" "identifier" = "04007F00050202000000010001" "changeTime" = "1008271213222B0800" "serviceIdentifier" = "54" "sivolumeUplink" = "711C" "sivolumeDownlink" = "09B2C5" "serviceIdentifier" = "62" "sivolumeUplink" = "05D8" "sivolumeDownlink" = "0340" "serviceClass" = "62" "volumeUplink" = "05D8" "volumeDownlink" = "0340" "serviceClass" = "4B" "volumeUplink" = "711C" "volumeDownlink" = "09B2C5" "apnSelectionMode" = "00" "servedMSISDN" = "198612398426688" "chargingCharacteristics" = "0800" "chChSelectionMode" = "00" "sgsnPLMNIdentifier" = "525F01" "servedIMEI" = "0121610002016205" "rATType" = "01" "localSequenceNumber" = "0DDEF73B" End of Record (1)

 

A 样式到 B 样式的转换要点如下：

1.     header 和 trailer 都可直接转换，但是 B 样式需要加上 CDR 的计数

2.     B 样式的 header 还含有 record type ，需要从 A 样式的某个注释行中获取

3.     字段域的转换比较简单，都是一行对一行

4.     A 样式有注释行，转换时要全部删除

 

转换由 awk 和 sed 完成，完整的命令是这样的：

awk 'BEGIN { cnt=0; } { if ( $0 == "RECORD" ) { cnt++; printf("Record (%d)/n", cnt); } else if ( $0 == "." ) { printf("End of Record (%d)/n", cnt); } else { print $0; } }' layout_a | sed '/^Record/{N;N;N;s//n//g;}' | sed 's/^/(Record (.*)/).*input_type /(.*/)$//1 "/2"/;s/^F /(.*/) /(.*/)$/   "/1" = "/2"/;/^#/d' > layout b

 

具体解析如下：

1.    awk 'BEGIN { cnt=0; } { if ( $0 == "RECORD" ) { cnt++; printf("Record (%d)/n", cnt); } else if ( $0 == "." ) { printf("End of Record (%d)/n", cnt); } else { print $0; } }' layout_a | sed '/^Record/{N;N;N;s//n//g;}' | sed 's/^/(Record (.*)/).*input_type /(.*/)$//1 "/2"/;s/^F /(.*/) /(.*/)$/   "/1" = "/2"/;/^#/d' > layout b

判断 CDR 的 header 和 trailer 。如果是 header ，即该行内容为 ”RECORD” ，计数器加一，且输出 ”Record (?)” 到下一步；如果是 trailer ，即该行内容为 ”.” ，输出 ”End of Record (?)” 到下一步；否则，不作修改输出到下一步。

2.    awk 'BEGIN { cnt=0; } { if ( $0 == "RECORD" ) { cnt++; printf("Record (%d)/n", cnt); } else if ( $0 == "." ) { printf("End of Record (%d)/n", cnt); } else { print $0; } }' layout_a | sed '/^Record/{N;N;N;s//n//g;}' | sed 's/^/(Record (.*)/).*input_type /(.*/)$//1 "/2"/;s/^F /(.*/) /(.*/)$/   "/1" = "/2"/;/^#/d' > layout b

遇到 header ，合并后续三行。因为第四行保留了该 CDR 的 Record type 。

3.    awk 'BEGIN { cnt=0; } { if ( $0 == "RECORD" ) { cnt++; printf("Record (%d)/n", cnt); } else if ( $0 == "." ) { printf("End of Record (%d)/n", cnt); } else { print $0; } }' layout_a | sed '/^Record/{N;N;N;s//n//g;}' | sed ' s/^/(Record (.*)/).*input_type /(.*/)$//1 "/2"/ ; s/^F /(.*/) /(.*/)$/   "/1" = "/2"/;/^#/d' > layout b

从上一步生成的 header 提取合适的内容，形成 B 样式的 header 。注意括号的运用。

4.    awk 'BEGIN { cnt=0; } { if ( $0 == "RECORD" ) { cnt++; printf("Record (%d)/n", cnt); } else if ( $0 == "." ) { printf("End of Record (%d)/n", cnt); } else { print $0; } }' layout_a | sed '/^Record/{N;N;N;s//n//g;}' | sed 's/^/(Record (.*)/).*input_type /(.*/)$//1 "/2"/; s/^F /(.*/) /(.*/)$/   "/1" = "/2"/; /^#/d' > layout b

提取各个字段的键和值，并加上双引号，形成 B 样式的字段域。同样注意括号的运用。

5.    awk 'BEGIN { cnt=0; } { if ( $0 == "RECORD" ) { cnt++; printf("Record (%d)/n", cnt); } else if ( $0 == "." ) { printf("End of Record (%d)/n", cnt); } else { print $0; } }' layout_a | sed '/^Record/{N;N;N;s//n//g;}' | sed 's/^/(Record (.*)/).*input_type /(.*/)$//1 "/2"/;s/^F /(.*/) /(.*/)$/   "/1" = "/2"/; /^#/d ' > layout b

删除剩余的注释行。