运用TF-IDF 提取特征词 k-means 挖掘微博的兴趣特征群体



//分词

#include "TFIDFMeasure.h"
#include <limits>
#include <cmath>
using namespace std;

TFIDFMeasure::~TFIDFMeasure(void)
{
 //销毁分词器
 if (this->_tokenizer!=NULL)
 {
  delete _tokenizer;
  _tokenizer = NULL;
 }
 //清空数据
 _docs.clear();
 _terms.clear();
 _wordsIndex.clear();
}

TFIDFMeasure::TFIDFMeasure(const StrVec& documents,ITokeniser* tokeniser)
{
 _docs=documents;
 _numDocs=documents.size();
 _tokenizer = tokeniser;
 this->Init();
}

void TFIDFMeasure::GenerateTerms(const StrVec& docs,StrVec& terms)
{
 for (int i=0; i < docs.size() ; i++)
 {
  StrVec words;
  _tokenizer->Partition(docs[i],words);//分词  

  for (int j=0; j < words.size(); j++)
  {
   //不在单词表中，则加入
   if (find(terms.begin(),terms.end(),words[j])==terms.end())
   {
    terms.push_back(words[j]);
   }
  }
 }
}

void TFIDFMeasure::Init()
{//初始化
 this->GenerateTerms (_docs,_terms);//分出所有词项
 this->_numTerms=_terms.size() ;//所有文档中的词项数目
 //准备好存储空间
 _maxTermFreq.resize(_numDocs);
 _docFreq.resize(_numTerms);
 _termFreq.resize(_numTerms);
 _termWeight.resize(_numTerms);

 for(int i=0; i < _terms.size() ; i++)   
 {
  _termWeight[i].resize(_numDocs);
  _termFreq[i].resize(_numDocs) ;
  _wordsIndex[_terms[i]] = i;//将单词放入单词映射表中 
 }
 this->GenerateTermFrequency ();//计算单词频率
 this->GenerateTermWeight();//计算单词权重
}

void TFIDFMeasure::GetWordFrequency(string& input,map<string,int>& freq)
{//计算单词频率
 transform(input.begin(),input.end(),input.begin(),tolower);
 StrVec temp;
 this->_tokenizer->Partition(input,temp);//对当前文档分词
 unique(temp.begin(),temp.end());
 StrVec::iterator iter;
 for (iter=temp.begin();iter!=temp.end();++iter)
 {
  int count = CountWords(*iter, temp);//计算单词在文档中出现的次数
  freq[*iter] = count;//保存单词频率
 }
} 
void  TFIDFMeasure::GetTermVector(int doc,DoubleVec& vec)
{
 vec.resize(this->_numTerms);
 for (int i=0; i < this->_numTerms; i++)           
  vec[i]=_termWeight[i][doc];//第i个单词在文档doc中的权重
}

//用于字符串比较的仿函数
class WordComp
{
public:
 WordComp(string& sWord) : word(sWord)
   {

   }
   bool operator() (const string& lhs)
   {
    return lhs.compare(word)==0;
   }      
private:
 string word;       
};

int TFIDFMeasure::CountWords(string& word, const StrVec& words)
{
 int nCount = 0;
 nCount = count_if(words.begin(),words.end(),WordComp(word));
 return nCount;
} 

int TFIDFMeasure::GetTermIndex(const string& term)
{
 map<string,int>::iterator pos = _wordsIndex.find(term);
 if (pos!=_wordsIndex.end())
 {
  return pos->second;
 }
 else
  return -1;
}

void TFIDFMeasure::GenerateTermFrequency()
{//计算每个单词在每份文档出现的频率
 for(int i=0; i < _numDocs  ; i++)
 {        
  string curDoc=_docs[i];//当前待处理的文档
  map<string,int> freq;
  this->GetWordFrequency(curDoc,freq);
  map<string,int>::iterator iter;
  _maxTermFreq[i]=numeric_limits<int>::min();
  for (iter = freq.begin();iter!=freq.end();++iter)
  {
   string word=iter->first;
   int wordFreq=iter->second ;
   int termIndex=GetTermIndex(word);//单词下标
   if(termIndex == -1)
    continue;
   _termFreq [termIndex][i]=wordFreq;//单词在第i份文档中出现的频率
   _docFreq[termIndex]++;//出现第termIndex单词的文档频率加1

   if (wordFreq > _maxTermFreq[i]) _maxTermFreq[i]=wordFreq;//记录第i份文档中的最大词频     
  }
 }
}

void TFIDFMeasure::GenerateTermWeight()
{//计算每个单词在每份文档中的权重   
 for(int i=0; i < _numTerms; i++)
 {
  for(int j=0; j < _numDocs ; j++)
  {
   _termWeight[i][j]=ComputeTermWeight (i, j);  
  }
 }
}

double TFIDFMeasure::GetTermFrequency(int term, int doc)
{   
 int freq=_termFreq [term][doc];//词频
 int maxfreq=_maxTermFreq[doc];   
 return ( (float) freq/(float)maxfreq );
}

double TFIDFMeasure::ComputeTermWeight(int term, int doc)
{//计算单词在文档中的权重
 float tf=GetTermFrequency (term, doc);
 float idf=GetInverseDocumentFrequency(term);
 return tf * idf;
}
double TFIDFMeasure::GetInverseDocumentFrequency(int term)
{
 int df=_docFreq[term];//包含单词term的文档数目
 return log((float) (_numDocs) / (float) df );
}

//k-means

#include "KMeans.h"
#include <time.h>
#include "Cluster.h"
#include "TermVector.h"
#include <limits>

KMeans::KMeans(Double2DVec &data, int K)
{
 int i;
 this->_coordinates.resize(data.size());
 for (i=0;i<data.size();++i)
 {
  copy(data[i].begin(),data[i].end(),back_inserter(_coordinates[i]));
 }
 _coordCount = data.size();
 _k = K;
 _clusters.resize(K);
 _clusterAssignments.resize(_coordCount);
 _nearestCluster.resize(_coordCount);
 _distanceCache.resize(_coordCount);
 for (i=0;i<_coordCount;++i)
 {
  _distanceCache[i].resize(_coordCount);
 }
 InitRandom();
}

void KMeans::InitRandom()
{
 srand(unsigned(time(NULL)));
 for (int i = 0; i < _k; i++)
 {
  int temp =  rand()%(_coordCount);//产生随机数
  _clusterAssignments[temp] = i; //记录第temp个资料属于第i个聚类
  _clusters[i] = new Cluster(temp,_coordinates[temp]);
 }
}
void KMeans::Start()
{
 int iter = 0,i,j;
 while (true)
 {
  
  cout<<"Iteration "<<iter++<< "..."<<endl;
  //1、重新计算每个聚类的均值
  for (i = 0; i < _k; i++)
  {
   _clusters[i]->UpdateMean(_coordinates);
  }

  //2、计算每个数据和每个聚类中心的距离
  for (i = 0; i < _coordCount; i++)
  {
   for (j = 0; j < _k; j++)
   {
    double dist = getDistance(_coordinates[i], _clusters[j]->Mean);
    _distanceCache[i][j] = dist;
   }
  }

  //3、计算每个数据离哪个聚类最近
  for (i = 0; i < _coordCount; i++)
  {
   _nearestCluster[i] = this->NearestCluster(i);
  }

  //4、比较每个数据最近的聚类是否就是它所属的聚类
  //如果全相等表示所有的点已经是最佳距离了，直接返回；
  int k = 0;
  for (i = 0; i < _coordCount; i++)
  {
   if (_nearestCluster[i] == _clusterAssignments[i])
    k++;

  }
  if (k == _coordCount)
   break;

  //5、否则需要重新调整资料点和群聚类的关系，调整完毕后再重新开始循环；
  //需要修改每个聚类的成员和表示某个数据属于哪个聚类的变量
  for (j = 0; j < _k; j++)
  {
   _clusters[j]->CurrentMembership.clear();
  }
  for (i = 0; i < _coordCount; i++)
  {
   _clusters[_nearestCluster[i]]->CurrentMembership.push_back(i);
   _clusterAssignments[i] = _nearestCluster[i];
  }

 }

}

double KMeans::getDistance(const DoubleVec& coord, const DoubleVec& center)
{
 return 1- TermVector::ComputeCosineSimilarity(coord, center);
}
int KMeans::NearestCluster(int ndx)
{
 int nearest = -1;
 double min = numeric_limits<double>::max();
 for (int c = 0; c < _k; c++)
 {
  double d = _distanceCache[ndx][c];
  if (d < min)
  {
   min = d;
   nearest = c;
  }

 }
 return nearest;
}
KMeans::~KMeans(void)
{
 vector<Cluster*>::iterator iter;
 for (iter=this->_clusters.begin();iter!=_clusters.end();++iter)
 {
  delete (*iter);
 }
 _clusters.clear();
}