linux去重某一列,Linux shell命令统计某列去重后的值

需求：根据mac获取，mac对应的硬件版本和软件版本。在根据硬件版本统计mac的数量

$ head test_1.txt

00:07:67,ec2108,hwv010537p0000

00:07:63,z83,znv861010p1225

00:07:67,ec6108v9u_ca_sccdx,hwv218012p0000

00:07:63,sbox8900hisi3716v3,zgv865012p0001

00:07:67,ec6108v9u_ca_sccdx,hwv218012p0000

00:21:26,ec6108v9u_ca_sccdx,hwv218012p0000

00:07:67,ec6108v9u_ca_sccdx,hwv218012p0000

00:07:63,stbhd-st7105-000,ztv70433089p0002

00:21:26,ec6108v9u_ca_sccdx,hwv215012p0000

00:21:26,ec6108v9u_ca_sccdx,hwv218012p0000

第二列去重

$ cat test_2.txt | awk -f ',' '{print $2}' | sort | uniq

ec2108

ec6108v9u_ca_sccdx

sbox8900hisi3716v3

stbhd-st7105-000

z83

将字符串转化为列表

import os

path = "c:\\users\\administrator\\desktop\\stb\\dd.txt"

read = open(path,"r")

list_ = []

for line in read:

line_ = line.strip()

list_.append(line_)

print(list_)

根据厂商和硬件版本获取对应的软件版本

方法一：

get gather-012-20180109/_search

{

"size": 0,

"aggs": {

"1": {

"terms": {

"field": "dev_hard_ver",

"size": 200

},

"aggs": {

"2": {

"terms": {

"field": "soft_ver",

"size": 1

}

}

}

}

}

}

方法二：

import os

from elasticsearch import elasticsearch

path = "c:\\users\\administrator\\desktop\\stb\\fact_stb.txt"

write_file = open(path,"a+")

es = elasticsearch()

def get_data(dev_hard_ver,dev_vendor_name):

index_ = "gather-012-20180109"

try:

rs = es.search(index=index_,body={

"size": 1,

"query": {

"bool": {

"must": [

{"term": {

"dev_hard_ver": {

"value": dev_hard_ver

}

}

},

{

"term": {

"dev_vendor_name": {

"value": dev_vendor_name

}

}

}

]

}

},

"_source": ["soft_ver","dev_vendor_name","dev_hard_ver"]

})

return rs

except:

return -1

dev_hard_ver=["02l0b61701mtas0001", "0x209", "0x441", "0x485", "0xe0a", "0xe17", "16060102l0b61701mtas0001", "1828", "1903", "2035", "5202", "99007002l0b6170635100791", "9900b002l0b61701iho3000t", "990104010004102300000001", "99010402l0b6083100000001", "990501010003170908a5c800", "a7.32.64.a0", "b860av2.1", "bjds2z-52", "chep3000sc", "chi619112", "chiho3000", "chiho3000a", "chiho3000jx2", "chiho33", "chiho3300a", "chiho3300ad", "cub-androidstb", "ec2106v1", "ec2108", "ec6108v9_pub_gdydx", "ec6108v9u_ca_sccdx", "ec6108v9u_pub_hbjdx", "ec6108v9u_pub_sccdx", "g2-40_2544", "hg510i", "hg650-tdswh", "hg650-tdswh-11", "ht670-v", "hy_v1.0", "hy107", "hy112", "hy127", "hy2021", "hy34e71c02l0b6v003hg510p04", "hy34e71c02l0b6v3.0hg510p10", "hy44170102l0b6v003hg510p04", "hy9900e0013798kk0120170001", "hy990104010004102300000001", "hyc37.e", "hychep920sc", "hychep921sc", "hyee", "hyhg510iv1.0", "hyhg510pv3.0", "hyty_h1.00.00", "hyv10-androidigw-068", "hyyix-g210v1.1", "ibhn-androidstb-068", "ibhr-androidstbga-069", "ibhu-androidstb-068", "ibhx-androidstbga-068", "ibhx-androidstbga-069", "iptv-s63-v1.1", "iptv-s65-v1.1", "iptv-s65-v1.2", "itv628 hd", "kb1.100.01h", "m8043v02", "mr222-dsgh-19", "mx_ca_sccdx", "s80-e-r1.01.01", "sbox8900hisi3716v3", "sbox8900hisi3716v386501007", "stbab-androidstb-068", "stbab-androidstbga-069", "stbhd-aml8626x-000", "stbhd-st7105-000", "stbhd-st7105-128-256-000", "stbho-aml8626x-000", "tgh0001", "ty_a1.00.00", "ty_h1.00.00", "v1", "v10-androidigw-068", "v2.1", "z82", "z83", "z85e", "zxb700v5c(ibhl-000-000)", "zxb860a(ibhr-androidstb-068)", "zxb860a(ibhr-androidstbga-068)"]

dev_vendor_name=["scty", "changhong", "jiuzhou", "jiuzhou", "hisense", "hisense", "scty", "skyworth", "skyworth", "skyworth", "huawei", "zte", "changhong", "skyworth", "skyworth", "zhaoneng", "utstarcom", "zte", "fiberhome", "changhong", "changhong", "changhong", "changhong", "changhong", "changhong", "changhong", "changhong", "youhua", "huawei", "huawei", "huawei", "huawei", "huawei", "huawei", "yinhe", "utstarcom", "fiberhome", "fiberhome", "zhaoneng", "scty", "hisense", "hisense", "hisense", "skyworth", "yhtc", "yhtc", "yhtc", "fiberhome", "skyworth", "huawei", "changhong", "changhong", "hisense", "utstarcom", "yhtc", "scty", "zte", "ccs", "zte", "zte", "zte", "zte", "zte", "chinagci", "chinagci", "chinagci", "changhong", "30kt", "huawei", "fiberhome", "huawei", "chinagci", "sunniwell", "sunniwell", "zte", "zte", "zte", "zte", "zte", "zte", "yihekeji", "scty", "scty", "zte", "zte", "fiberhome", "zhaoneng", "zhaoneng", "zhaoneng", "zte", "zte", "zte"]

for i in range(len(dev_vendor_name)):

rs = get_data(dev_hard_ver[i],dev_vendor_name[i])

if rs!=-1:

for hit in rs['hits']['hits']:

soft_ver=hit['_source']['soft_ver']

if soft_ver is none:

soft_ver='none'

write_file.write(soft_ver+"\n")

else:

write_file.write("-1" + "\n")

write_file.close()

总结

以上就是这篇文章的全部内容了，希望本文的内容对大家的学习或者工作具有一定的参考学习价值，谢谢大家对萬仟网的支持。如果你想了解更多相关内容请查看下面相关链接