/*默认为重新建立LSH index。
**使用--index命令来保存LSH index。
**下一次运行mplsh-run.cpp时,使用 --index命令会加载上次保存的index;此时需要**确保数据和参数与上次运行时相同,不过 benchmark file ,Q 和K 可以不同。
*/
int main (int argc, char *argv[])
{
...
if (vm.count("index") == 1) {
use_index = true;
}
...
cout << "LOADING DATA..." << endl;
FloatMatrix data(data_file);
//1.从路径加载数据集,data(data_file)使用的是Matrix的自定义构造函数:
/// Construct from a file.
Matrix (const std::string &path): dims(NULL) { load(path); }
typedef MultiProbeLshIndex<unsigned> Index;
//使用的是p-stable LSH,还要经过二次哈希,使用LSH的索引结构
/*template <typename KEY>
**class MultiProbeLshIndex: public LshIndex<MultiProbeLsh, KEY>
*/
//class MultiProbeLsh: public RepeatHash<GaussianLsh>
//typedef StableDistLsh<Gaussian> GaussianLsh; --lsh.h--
//typedef boost::normal_distribution<float> Gaussian; --common.h--
FloatMatrix::Accessor accessor(data);
//访问数据集,data是FloatMatrix的对象,可以理解为一维数组;
//accessor(data),调用Accessor的构造函数:
/** Accessor(const Matrix &matrix)
: matrix_(matrix), flags_(matrix.getSize()) {} **/
Index index;
// try loading index
bool index_loaded = false;
if (use_index) {
//使用--index命令后,则置use_index =true,加载上次保存的index
ifstream is(index_file.c_str(), ios_base::binary);
if (is) {
...
index.load(is);
index_loaded = true;//index加载完毕
}
}
if (!index_loaded) {
/**默认设置是重新建立索引;第一次运行程序的时候还没有index,必须要建立,然后下一次使用的时候就可以通过--index命令来从data_file加载这个index ***/
// We define a short name for the MPLSH index.
Index::Parameter param;
// Setup the parameters. Note that L is not provided here.
param.W = W;
param.range = H;
// See H in the program parameters. You can just use the default value.
param.repeat = M;
param.dim = data.getDim();
DefaultRng rng;
-------------------------------------------------------------------------------------------------------------------------------------------
index.init(param, rng, L);
/*init是mplsh.h中Ln295MultiProbeLshIndex中的init:
template <typename Engine>
void init (const Parameter ¶m, Engine &engine, unsigned L) {
Super::init(param, engine, L);//lsh-index.h 设置了lsh_,tables_
param_ = param;
// we are going to normalize the distance by window size, so here we pass W = 1.0.
// We tune adaptive probing for KNN distance range [0.0001W, 20W].
recall_.reset(MultiProbeLshModel(Super::lshs_.size(), 1.0, param_.repeat, Probe::MAX_T), 200, 0.0001, 20.0);
}
*recall_.reset先不管吧。
*这里面又使用了super::init,指的是lsh-index.h中的init:
template <typename Engine>
void init (const Parameter ¶m, Engine &engine, unsigned L)
{
BOOST_VERIFY(lshs_.size() == 0);
BOOST_VERIFY(tables_.size() == 0);
lshs_.resize(L);
tables_.resize(L);
for (unsigned i = 0; i < L; ++i) {
lshs_[i].reset(param, engine);
...
tables_[i].resize(lshs_[i].getRange());
}
}
*这里面的lshs_[i].reset(...),因为lshs_[i]的类型是MultiProbeLsh,因而此处的reset()是指mplsh.h 中class MultiProbeLsh中的reset() --Ln228:
template <typename RNG>
void reset(const Parameter ¶m, RNG &rng)
{
H_ = param.range;
Super::reset(param, rng);//lsh_[i].reset(param, rng); a_[i] = rng()
}
*此处的reset()是文件composite.h中的class RepeatHash里面的reset()--Ln392:
template <typename RNG>
void reset(const Parameter ¶m, RNG &rng)
{
assert(param.repeat > 0);
lsh_.resize(param.repeat);
for (unsigned i = 0; i < param.repeat; ++i)
{
lsh_[i].reset(param, rng);//std::vector<LSH> lsh_;
}
a_.resize(param.repeat); //std::vector<unsigned> a_;
for (unsigned i = 0; i < param.repeat; ++i) a_[i] = rng();
}
*因为此处的父类是RepeatHash类,使用的是RepeatHash<GaussianLsh>,GaussianLsh<==>StableDistLsh<Gaussian>,因而这里面的lsh_[i].reset()指的是StableDistLsh类中的reset():生成随机数a_,b_,
template <typename RNG>//RNG <==>Gaussian
void reset(const Parameter ¶m, RNG &rng)
{
a_.resize(param.dim); //std::vector<float> a_;
W_ = param.W;
dim_ = param.dim;
boost::variate_generator<RNG &, DIST> gen(rng, DIST());
for (unsigned i = 0; i < dim_; ++i) a_[i] = gen();
b_ = boost::variate_generator<RNG &, Uniform>(rng, Uniform(0,W_))();
}
*到这里reset就调用到底了;总体来说设置了b_,a_,W_,dim_,H_,及二次哈希要用到的随机数组a_[]。
*整体的哈希函数结构差不多清楚了,首先是哈希函数族数组lshs_[0~L],对应L个哈希表;每个lshs_[i]中包括了一个哈希函数数组lsh_[0~M]、二次哈希要用的随机数组a_[0~M]、和其它的参数如H_;
*tables_[0~L][0~H]对应L个哈希表;
-------------------------------------------------------------------------------------------------------------------------------------------
// The accessor.
// Initialize the index structure. Note L is passed here.
cout << "CONSTRUCTING INDEX..." << endl;
timer.restart();
{
boost::progress_display progress(data.getSize());
for (unsigned i = 0; i < data.getSize(); ++i)
{
// Insert an item to the hash table.
// Note that only the key is passed in here.
// MPLSH will get the feature from the accessor.
index.insert(i, data[i]);//(key , value)
/**************************************************************************调用lsh-index.h中的insert():
void insert (Key key, Domain value)
{
for (unsigned i = 0; i < lshs_.size(); ++i) {
//将lshs_中表示的每一个哈希函数应用的每一个vector,
unsigned index = lshs_[i](value);//一个向量都要经过每一个哈希函数族的映射;
------------------------------------------------------------------------------------------------------------------------------------------
这里lshs_[i](value)的括号是重载了RepeatHash中的括号,对哈希值再次哈希,代码如下:
unsigned operator () (Domain obj) const
{
unsigned ret = 0;
for (unsigned i = 0; i < lsh_.size(); ++i)
{
ret += lsh_[i](obj) * a_[i];//二次哈希(a1*h1 + a2*h2 + aN*hN);第一次的哈希值直接进行二次计算;
}//lsh.h Ln162 floor((a*v+b)/W)
return ret;
}
这里的lsh_[i](obj)又是括号的重载,是StableDistHash<Gaussian>中的,计算哈希值:
unsigned operator () (Domain obj) const
{
float ret = b_;
for (unsigned i = 0; i < dim_; ++i)
{
ret += a_[i] * obj[i];
}
return unsigned(int(std::floor(ret / W_)));
}
-------------------------------------------------------------------------------------------------------------------------------------------
再看insert(i, data[i]):data[i]使用了重载的中括号[],代码在matrix.h中,Ln97,用于访问第i个向量:
/// Access the ith vector.
const T *operator [] (int i) const {
return dims + i * dim;
}// T <==> float
综上,变量index = 对第i个向量二次哈希后的值,也就是该向量应放入的桶号吧;程序下一行就是将向量对应的key存入tables_的桶中;从mplsh-run中传过来的key值是i,可见向量的key是向量在数据集文件(data)中的编号(i=0~N);
****************************************************/
tables_[i][index].push_back(key);
}
}
//该句的作用是将key放入第i个哈希表的index号桶中。tables_是一个二维数组,每一个元素是Bin(存的是很多个key);到这里就把向量映射进哈希桶中了。
/* typedef std::vector<Key> Bin;
std::vector<LSH> lshs_;
std::vector<std::vector<Bin> > tables_;*/
** 哈希表填充完毕~~耶依~~ **
**************************************************/
++progress;
}
}
...
if (use_index) {
...
{
ofstream os(index_file.c_str(), ios_base::binary);
os.exceptions(ios_base::eofbit | ios_base::failbit | ios_base::badbit);
index.save(os);
}
cout << boost::format("SAVING TIME: %1%s") % timer.elapsed() << endl;
}
}
###############################################################
## 还有最后一部分了~~ ##########################################
/* bool do_benchmark = true;
...
if ((Q == 0) || (vm.count("benchmark") == 0)) {
do_benchmark = false;
}
*/
if (do_benchmark) {
//benchmark文件中每一行存储的是抽样出来的Q的查询点及它们的KNN点。
//<query ID> <K> <1st NN's ID> <distance> <2nd NN's ID> <distance> ... <Kth NN's ID> <distance>
Benchmark<> bench;// class Benchmark --eval.h --Ln 66
cout << "LOADING BENCHMARK..." << endl;
bench.load(benchmark);
--------------------------------------------------------------------
// void load (const std::string &path)
{
std::ifstream is(path.c_str());
load(is);
is.close();
}
void load (std::istream &is)
{
std::string line;
//调用clear之后, vector的尺寸(size,指元素个数)将变成zero. 但它的容量(capacity,占用的空间)却并不发生变化, vector本身并不释放任何内存.
queries_.clear();
topks_.clear();
for (;;) {//这段代码的作用是,提取向量的ID和该向量的KNN.参考benchfile每一行的结构:ID,K,1stID/dist,2ndID/dist,....
unsigned q, k;
is >> q;//设置q的值,表示向量的ID?
if (!is) break;
queries_.push_back(q);//把q添加到queries_的尾部
is >> k;//设置k
topks_.push_back(Topk<KEY>());
//在topks_最后添加一项Topk<KEY>,并设置大小为k,即包含k个TopkEntry。
topks_.back().resize(k);
//依次设置每个TopkEntry的key和dist域
for (unsigned j = 0; j < k; j++) {
is >> topks_.back()[j].key;
is >> topks_.back()[j].dist;
}
}
Q_ = queries_.size();//query的个数
}
--------------------------------------------------------------------
bench.resize(Q, K);
cout << "DONE." << endl;
for (unsigned i = 0; i < Q; ++i)
{
for (unsigned j = 0; j < K; ++j)
{
assert(bench.getAnswer(i)[j].key < data.getSize());
//getAnswer(i),返回 topks_[i];第i个query的第j-NN
}
}
cout << "RUNNING QUERIES..." << endl;
未完待续 ....
下一部分感觉关于ProbeSequence的有点麻烦,表示数学没学好,看不懂额 -_-|||