#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Sep 3 19:52:45 2017
@author: vicky
"""
class simhash:
#构造函数
def __init__(t, tokens='', hashbits=n):
t.hashbits = hashbits
t.hash = t.simhash(tokens);
#toString函数
def __str__(t):
return str(t.hash)
#生成simhash值
def simhash(t, tokens):
v = [0] * t.hashbits
for tmp in [t._string_hash(x) for x in tokens]: #t为token的普通hash值
for i in range(t.hashbits):
bitmask = 1 << i
if tmp & bitmask :
v[i]+=1 #查看当前bit位是否为1,是的话将该位+1
else:
v[i]-=1 #否则的话,该位-1
fingerprint=0
for i in range(t.hashbits):
if v[i]>=0:
fingerprint+=1<< i
return fingerprint #整个文档的fingerprint为最终各个位>=0的和
#求海明距离
def hamming_distance(t, other):
x = (t.hash ^ other.hash) & ((1 << t.hashbits) - 1)
tot = 0;
while x :
tot += 1
x &= x - 1
return tot
#求相似度
def similarity (t, other):
a = float(t.hash)
b = float(other.hash)
if a > b : return b/a
else: return a/b
# 针对source生成hash值
def _string_hash(t, source):
if source == "":
return 0
else:
x = ord(source[0]) << 7
m = 1000003
mask = 2 ** t.hashbits - 1
for c in source:
x = ((x * m) ^ ord(c)) & mask
x ^= len(source)
if x == -1:
x = -2
return x
import numpy as np
import pandas as pd
from sklearn.cluster import KMeans
n=size(test)
#输入地址数据t,n=样本数量
#去掉样本中的特定中文字符,以免影响相似度的计算
m=''
test2=test
for i in range(n):
m=re.split(u'省|市|区|镇|村|路|街|栋|单元|层|号|楼|室', test[i])
test2[i]=''.join(m)
dis=np.zeros((n,n), dtype=np.int)
sim=np.zeros((n,n))
#计算测试集两两之间的文本相似度
for i in range(n):
for j in range(n):
hash1 = simhash(test2[i].split())
hash2 = simhash(test2[j].split())
dis[i,j]=hash1.hamming_distance(hash2)
sim[i,j]=hash1.similarity(hash2)
#去重
test3=test
for j in range(n):
for i in range(j):
if sim[i,j]>0.8 and dis[i,j]<3:
test3[j]=0
#文本聚类:根据hamming距离矩阵dis做k均值聚类
km = KMeans(n_clusters=16).fit(dis)
c=km.cluster_centers_
#每个样本所属的簇
l=km.labels_
df.to_excel('SomeTable.xls', index=False)
import csv
with open('test2.csv', 'wb') as csvfile:
#spamwriter = csv.writer(csvfile, delimiter=' ',quotechar='|', quoting=csv.QUOTE_MINIMAL)
spamwriter = csv.writer(test2, dialect='excel')
spamwriter.writerow(['Spam'] * 5 + ['Baked Beans'])
spamwriter.writerow(['Spam', 'Lovely Spam', 'Wonderful Spam'])
np.savetxt('test1.out', (test,test2,test3)) #x,y,z相同大小的一维数组