逆向文件频率(inverse document frequency,IDF)是一个词语普遍重要性的度量。某一特定词语的IDF,可以由总文件数目除以包含该词语之文件的数目,再将得到的商取对数.
IDF i = log N / n i
N 是代表语料库中文件数量的总数, n i 是代表包含词语n i 的文件数目所包含的词 i 的总数.
class IDFs
{
HashMap<String,Double> IDFsList = new HashMap<String, Double>();
ArrayList<ArrayList<String>> IDFsMainFileList = new ArrayList<ArrayList<String>>();
ArrayList<String> IDFsave = new ArrayList<String>();
int Ncount;
public IDFs(ArrayList<ArrayList<String>> idf)
{
IDFsMainFileList = idf;
Ncount = IDFsMainFileList.size();
}
public HashMap<String,Double> PrintIDFs()
{
for(int i=0; i<IDFsMainFileList.size(); i++)
{
ArrayList<String> IDFsSubFileList = IDFsMainFileList.get(i);
ArrayList<String> list = new ArrayList<String>();
for(int j=0; j<IDFsSubFileList.size(); j++)
{
if(!list.contains(IDFsSubFileList.get(j)))
{
list.add(IDFsSubFileList.get(j));
//Take elements from arraylist<arraylist<string>>
if(!IDFsList.containsKey(IDFsSubFileList.get(j)))
{
IDFsList.put(IDFsSubFileList.get(j),1.0);//save elements to arraylist<hashmap<String,Double>>
IDFsave.add(IDFsSubFileList.get(j));//take elements from hashmap
}
else
{
double value = IDFsList.get(IDFsSubFileList.get(j));
value++;
IDFsList.put(IDFsSubFileList.get(j),value);
}
}
}
}
for(int k=0; k<IDFsave.size(); k++)
{
double nTotal = IDFsList.get(IDFsave.get(k));
double temp = Ncount / nTotal;
double idfs = Math.log(temp);
IDFsList.put(IDFsave.get(k), idfs);
}
return IDFsList;
}
}