address2.py-20170905

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Sep  3 19:52:45 2017

@author: vicky
"""

class simhash:       
    #构造函数  
    def __init__(t, tokens='', hashbits=n):         
        t.hashbits = hashbits  
        t.hash = t.simhash(tokens);      
    #toString函数     
    def __str__(t):  
        return str(t.hash)   
    #生成simhash值     
    def simhash(t, tokens):  
        v = [0] * t.hashbits  
        for tmp in [t._string_hash(x) for x in tokens]: #t为token的普通hash值            
            for i in range(t.hashbits):  
                bitmask = 1 << i  
                if tmp & bitmask :  
                    v[i]+=1 #查看当前bit位是否为1,是的话将该位+1  
                else:  
                    v[i]-=1 #否则的话,该位-1  
        fingerprint=0  
        for i in range(t.hashbits):  
            if v[i]>=0:  
                fingerprint+=1<< i  
        return fingerprint #整个文档的fingerprint为最终各个位>=0的和       
    #求海明距离  
    def hamming_distance(t, other):  
        x = (t.hash ^ other.hash) & ((1 << t.hashbits) - 1)  
        tot = 0;  
        while x :  
            tot += 1  
            x &= x - 1  
        return tot    
    #求相似度  
    def similarity (t, other):  
        a = float(t.hash)  
        b = float(other.hash)  
        if a > b : return b/a  
        else: return a/b  
   # 针对source生成hash值
    def _string_hash(t, source):         
        if source == "":  
            return 0  
        else:  
            x = ord(source[0]) << 7  
            m = 1000003  
            mask = 2 ** t.hashbits - 1  
            for c in source:  
                x = ((x * m) ^ ord(c)) & mask  
            x ^= len(source)  
            if x == -1:  
                x = -2  
            return x  


import numpy as np
import pandas as pd
from sklearn.cluster import KMeans  

n=size(test) 
#输入地址数据t,n＝样本数量

#去掉样本中的特定中文字符，以免影响相似度的计算
m=''
test2=test
for i in range(n):
    m=re.split(u'省|市|区|镇|村|路|街|栋|单元|层|号|楼|室', test[i]) 
    test2[i]=''.join(m)

dis=np.zeros((n,n), dtype=np.int)
sim=np.zeros((n,n))
#计算测试集两两之间的文本相似度
for i in range(n):
    for j in range(n):
        hash1 = simhash(test2[i].split())  
        hash2 = simhash(test2[j].split())  
        dis[i,j]=hash1.hamming_distance(hash2)
        sim[i,j]=hash1.similarity(hash2)

#去重
test3=test
for j in range(n):
    for i in range(j):
        if sim[i,j]>0.8 and dis[i,j]<3:
            test3[j]=0
                  
#文本聚类:根据hamming距离矩阵dis做k均值聚类         
km = KMeans(n_clusters=16).fit(dis)  
c=km.cluster_centers_ 
#每个样本所属的簇  
l=km.labels_   
   

df.to_excel('SomeTable.xls', index=False)
       
import csv
with open('test2.csv', 'wb') as csvfile:
    #spamwriter = csv.writer(csvfile, delimiter=' ',quotechar='|', quoting=csv.QUOTE_MINIMAL)
    spamwriter = csv.writer(test2, dialect='excel')
    spamwriter.writerow(['Spam'] * 5 + ['Baked Beans'])
    spamwriter.writerow(['Spam', 'Lovely Spam', 'Wonderful Spam'])

np.savetxt('test1.out', (test,test2,test3))   #x,y,z相同大小的一维数组