awk qos分析脚本两例。
之前写的用于分析webcdn日志的awk脚本。一个可以用来分析流量和状态码。另一个用来分析错误码。
使用方式：
zcat /.log.gz |awk -f analyze_awk.awk - #domain和traffic相关分析，要注 意日志格式
zcat /.log.gz |awk -f auto_awk.awk - #wrong http code分析
（目前我们使用监控系统来直接调用脚本，在发现问题时可以直接分析错误的domain和url,方便快速定位问题）
有兴趣的同学测试下，欢迎大家共同学习和探讨。
脚本1：
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
|
{
gsub(/
"/,"
");
client=$1;
domain=$2;
url=$7;
status=$9;
bytes=$10;
upstream=$(NF-2);
time
=$NF;
if
(((match(status,
"4.."
))||(match(status,
"5.."
)))&& (status != 400)) {
++arr_status_num[status];
++arr_domain_num[domain];
++arr_domain_status[domain,status];
++arr_domain_url_num[
"http://"
domain
""
url
""
];
++arr_domain_url_status[
"http://"
domain
""
url
""
,status];
}
}
END {
#http status statistic
print
"\033[40;33m#################################\033[0m"
print
"\033[40;32m#get http code summary: code,sum#\033[0m"
print
"\033[40;33m#################################\033[0m"
status_sort=asorti(arr_status_num,sort_status);
for
(i=1; i<=status_sort; i++) {
s=sort_status[i];
if
(s ~
"4.."
) {
printf
"%s:%d\n"
,
s,arr_status_num[s];
}
else
if
(s ~
"5.."
) {
printf
"%s:%d\n"
,
s,arr_status_num[s];
}
}
#get domain statistics
print
"==================================================="
print
"\033[40;33m###############################################################\033[0m"
print
"\033[40;32m#get domain summary: domain|error_sum|4xx|5xx|404|499|502|504#\033[0m"
print
"\033[40;33m###############################################################\033[0m"
domain_sort=asorti(arr_domain_num,sort_domain);
for
(i=1; i<=domain_sort; i++) {
g=sort_domain[i];
arr_domain_num_error=arr_domain_status[g,502]+arr_domain_status[g,504]+arr_domain_status[g,404]+arr_domain_status[g,499]+arr_domain_status[g,415]+arr_domain_status[g,403];
arr_domain_num_4xx=arr_domain_status[g,404]+arr_domain_status[g,499]+arr_domain_status[g,415]+arr_domain_status[g,403]
arr_domain_num_5xx=arr_domain_status[g,502]+arr_domain_status[g,504]
printf
"%s|%d|%d|%d|%d|%d|%d|%d\n"
,
g,arr_domain_num_error,arr_domain_num_4xx,arr_domain_num_5xx,arr_domain_status[g,404],arr_domain_status[g,499],arr_domain_status[g,502],arr_domain_status[g,504] |
"sort -t '|' -k 2 -nr|head -20"
}
close(
"sort -t '|' -k 2 -nr|head -20"
)
#get url statistics
print
"==================================================="
print
"\033[40;33m################################################################\033[0m"
print
"\033[40;32m#get url summary: url | error_sum| 4xx| 5xx| 404| 499| 502| 504#\033[0m"
print
"\033[40;33m################################################################\033[0m"
url_sort=asorti(arr_domain_url_num,url_domain);
for
(i=1; i<=url_sort; i++) {
g=url_domain[i];
arr_domain_url_num_error=arr_domain_url_status[g,404]+arr_domain_url_status[g,499]+arr_domain_url_status[g,502]+arr_domain_url_status[g,504]+arr_domain_url_status[g,403]+arr_domain_url_status[g,415];
arr_domain_url_num_4xx=arr_domain_url_status[g,404]+arr_domain_url_status[g,499]+arr_domain_url_status[g,403]+arr_domain_url_status[g,415];
arr_domain_url_num_5xx=arr_domain_url_status[g,502]+arr_domain_url_status[g,504];
printf
"%s | %d| %d| %d| %d| %d| %d| %d\n"
,
g,arr_domain_url_num_error,arr_domain_url_num_4xx,arr_domain_url_num_5xx,arr_domain_url_status[g,404],arr_domain_url_status[g,499],arr_domain_url_status[g,502],arr_domain_url_status[g,504]|
"sort -t ' ' -k 3 -nr|head -20"
}
close(
"sort -t ' ' -k 3 -nr|head -20"
)
}
|
脚本2：
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
|
BEGIN {
OFS=
";"
}
{
gsub(/
"/,"
");
client=$1;
domain=$2;
url=$7;
status=$9;
bytes=$10;
upstream=$(NF-2);
time
=$NF;
if
(status>0 && (match(status,
"..."
))) {
sum_access++;
++arr_status_num[status];
arr_status_bytes[status]+=bytes;
arr_status_time[status]+=
time
;
++arr_domain_num[domain];
++arr_domain_status[domain,status];
++arr_domain_url_num[
"http://"
domain
""
url
""
];
++arr_domain_url_status[
"http://"
domain
""
url
""
,status];
if
(status==200 || status==206) {
#about domain
arr_domain_bytes[domain]+=bytes;
arr_domain_time[domain]+=
time
;
#about client(usualy ip)
arr_client_bytes[client]+=bytes;
arr_client_time[client]+=
time
;
if
(
time
<= 0.1) {
++arr_status_speed1[status]; ++arr_domain_speed1[domain];
}
if
((
time
<= 0.2) && (
time
>= 0.1)) {
++arr_status_speed2[status]; ++arr_domain_speed2[domain];
}
if
((
time
<= 0.5) && (
time
>= 0.2)) {
++arr_status_speed5[status]; ++arr_domain_speed5[domain];
}
if
((
time
<= 1) && (
time
>= 0.5)) {
++arr_status_speed10[status]; ++arr_domain_speed10[domain];
}
if
((
time
<= 2) && (
time
>= 1)) {
++arr_status_speed20[status]; ++arr_domain_speed20[domain];
}
if
((
time
<= 5) && (
time
>= 2)) {
++arr_status_speed50[status]; ++arr_domain_speed50[domain];
}
if
(
time
>= 5) {
++arr_status_speed5x[status]; ++arr_domain_speed5x[domain];
}
}
}
}
END {
#http status statistic
print
"\033[40;33m##########################################################################################################################\033[0m"
print
"\033[40;32m#get http code summary: code,sum,rate,speed(Kb),avg_time(s),<0.1 rate,<0.2 rate,<0.5 rate,<1 rate,<2 rate,<5 rate,>5 rate#\033[0m"
print
"\033[40;33m##########################################################################################################################\033[0m"
status_sort=asorti(arr_status_num,sort_status);
#print arr_status_time[200]
for
(i=1; i<=status_sort; i++) {
s=sort_status[i];
if
(s==200 || s==206) {
printf
"%s:%d,%.2f|%.2f|%.3f|%.2f,%.2f,%.2f,%.2f,%.2f,%.2f\n"
,
s,arr_status_num[s],arr_status_num[s]
/sum_access
,arr_status_bytes[s]*8/(1024*arr_status_time[s]),arr_status_time[s]
/arr_status_num
[s],
arr_status_speed1[s]
/arr_status_num
[s]*100,arr_status_speed2[s]
/arr_status_num
[s]*100,arr_status_speed5[s]
/arr_status_num
[s]*100,
arr_status_speed10[s]
/arr_status_num
[s]*100,arr_status_speed20[s]
/arr_status_num
[s]*100,arr_status_speed50[s]
/arr_status_num
[s]*100,arr_status_speed5x[s]
/arr_status_num
[s]*100;
}
else
if
(s ~
"3.."
) {
printf
"%s:%d,%.2f\n"
,
s,arr_status_num[s],arr_status_num[s]
/sum_access
;
}
else
if
(s ~
"4.."
) {
printf
"%s:%d,%.2f\n"
,
s,arr_status_num[s],arr_status_num[s]
/sum_access
;
}
else
if
(s ~
"5.."
) {
printf
"%s:%d,%.2f\n"
,
s,arr_status_num[s],arr_status_num[s]
/sum_access
;
}
}
#get domain statistics
print
"==================================================="
print
"\033[40;33m###########################################################################\033[0m"
print
"\033[40;32m#get domain summary: domain,sum,rate,valid rate,4xx rate,5xx rate,data(G)#\033[0m"
print
"\033[40;33m###########################################################################\033[0m"
domain_sort=asorti(arr_domain_num,sort_domain);
for
(i=1; i<=domain_sort; i++) {
g=sort_domain[i];
arr_domain_num_valid=arr_domain_status[g,200]+arr_domain_status[g,206]+arr_domain_status[g,302]+arr_domain_status[g,301];
arr_domain_num_4xx=arr_domain_status[g,404]+arr_domain_status[g,499]+arr_domain_status[g,403]+arr_domain_status[g,415];
# arr_domain_num_4xx=arr_domain_status[g,4..];
arr_domain_num_5xx=arr_domain_status[g,502]+arr_domain_status[g,504];
arr_domain_num_404=arr_domain_status[g,404]
arr_domain_num_499=arr_domain_status[g,499]
if
(arr_domain_time[g]>0) {
printf
"%s:%d,%.2f|%.2f,%.2f,%.2f,%.2f\n"
,
g,arr_domain_num[g],arr_domain_num[g]
/sum_access
,arr_domain_num_valid
/arr_domain_num
[g],arr_domain_num_4xx
/arr_domain_num
[g],arr_domain_num_5xx
/arr_domain_num
[g],arr_domain_bytes[g]*8/(1024*1024*1024) |
"sort -t ',' -k 2 -nr|head -20"
}
}
close(
"sort -t ',' -k 2 -nr|head -20"
)
#get url sum statistics
print
"==================================================="
print
"\033[40;33m###########################################################\033[0m"
print
"\033[40;32m#get url summary:url| sum| valid rate| 4xx rate |5xx rate#\033[0m"
print
"\033[40;33m###########################################################\033[0m"
url_sort=asorti(arr_domain_url_num,url_domain);
for
(i=1; i<=url_sort; i++) {
g=url_domain[i];
arr_domain_url_num_valid=arr_domain_url_status[g,200]+arr_domain_url_status[g,206]+arr_domain_url_status[g,302]+arr_domain_url_status[g,301];
arr_domain_url_num_4xx=arr_domain_url_status[g,404]+arr_domain_url_status[g,499]+arr_domain_url_status[g,403]+arr_domain_url_status[g,415];
arr_domain_url_num_5xx=arr_domain_url_status[g,502]+arr_domain_url_status[g,504];
arr_domain_url_num_404=arr_domain_url_status[g,404]
arr_domain_url_num_499=arr_domain_url_status[g,499]
printf
"%s| %d| %.2f| %.2f| %.2f\n"
,
g ,arr_domain_url_num[g] ,arr_domain_url_num_valid
/arr_domain_url_num
[g],arr_domain_url_num_4xx
/arr_domain_url_num
[g],arr_domain_url_num_5xx
/arr_domain_url_num
[g]|
"sort -t ' ' -k 2 -nr|head -20"
}
close(
"sort -t ' ' -k 2 -nr|head -20"
)
#get url errorsum statistics
print
"==================================================="
print
"\033[40;33m##########################################################################\033[0m"
print
"\033[40;32m#get url summary:url| sum| valid rate| errorsum rate| 4xx rate| 5xx rate#\033[0m"
print
"\033[40;33m##########################################################################\033[0m"
url_sort=asorti(arr_domain_url_num,url_domain);
for
(i=1; i<=url_sort; i++) {
g=url_domain[i];
arr_domain_url_num_valid=arr_domain_url_status[g,200]+arr_domain_url_status[g,206]+arr_domain_url_status[g,302]+arr_domain_url_status[g,301];
arr_domain_url_num_4xx=arr_domain_url_status[g,404]+arr_domain_url_status[g,499]+arr_domain_url_status[g,403]+arr_domain_url_status[g,415];
arr_domain_url_num_5xx=arr_domain_url_status[g,502]+arr_domain_url_status[g,504];
arr_domain_url_num_404=arr_domain_url_status[g,404]
arr_domain_url_num_499=arr_domain_url_status[g,499]
printf
"%s| %d| %.2f| %.2f| %.2f| %.2f\n"
,
g ,arr_domain_url_num[g] ,arr_domain_url_num_valid
/arr_domain_url_num
[g] ,(arr_domain_url_num_4xx+arr_domain_url_num_5xx)
/arr_domain_url_num
[g] ,arr_domain_url_num_4xx
/arr_domain_url_num
[g] ,arr_domain_url_num_5xx
/arr_domain_url_num
[g]|
"sort -t ' ' -k 4 -nr|head -10"
}
close(
"sort -t ' ' -k 4 -nr|head -10"
)
}
|
本文转自菜菜光 51CTO博客，原文链接：http://blog.51cto.com/caiguangguang/932359，如需转载请自行联系原作者
扫一扫
859
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
