matlab lsh,实习日记：图像检索算法 LSH 的总结与分析(matlab)

最开始仿真和精度测试，基于 matlab 完成的。

Demo_MakeTable.m (生成 Hash 表)

%========================================

%***********************************

%******* 设定参数: *****************

%******* l : hash表个数 ********

%******* k : 各表关键字个数 *******

%=========================================

clear all; close all; clc;

l = 3;

k = 15;

sData = textread(‘./Data/data.txt‘, ‘%s‘);

n = length(sData);

Data = false(n, 128);

for i = 1 : n

Data(i, :) = sData{i} - ‘0‘;

end

clear sData;

Tables = lsh_1norm(l, k, Data);

save([‘.\Data\‘ num2str(l) ‘_‘ num2str(k) ‘_Tables‘], ‘Tables‘);

% cd ./ErrorCompute

% Demo_ErrorStat

%% Write Hash keys and the Index

% 多线程

% matlabpool local 10

% for i = 1 : l

% fprintf(‘第 %d 个哈希表.\n‘, i);

% Write_Hash_Index(k, Tables(i), i);

% end

% matlabpool close

lsh_1norm.m (核心函数，我用其组织了整个索引结构生成过程)

function T2 = lsh_1norm(l, k, yy)

%==================================

% parameters setting && getting

% [n d] = size(x);

%==================================

%************ 数据集预处理 : 转 hamming 空间(维度小于100时使用) ********

% fprintf(‘数据集转换到 Hamming space.\n‘);

% tic

% x = x‘; % n x d (d < 100)

% C = max(x(:));

% dim = size(x);

% yy = false(dim(1), dim(2) * 255);

% for i = 1 : dim(1)

% for j = 1 : dim(2)

% oneO = false(1,C);

% oneO(1:x(i,j)) = 1;

% yy(i,(j-1)*C+1 : (j-1)*C+C) = oneO;

% end

% end

% clear oneO x;

% toc

% fprintf(‘转换 Hamming space 完成.\n‘);

%==================================

fprintf(‘初始化 %d 个 Hash 表...\n‘, l);

% matlabpool local 10

% 可并行

for i = 1 : l

% creat and init Tables[i] = f(k, x);

T1(i) = createTable(k, yy);

end

% matlabpool close

fprintf(‘初始化完成。\n‘);

%=====================================

tic;

matlabpool local 10

% save the index of feature data

% insert(T, x);

for i = 1 : l

fprintf(‘数据插入第 %d 个hash表\n‘, i);

T2(i) = insert_data(T1(i), yy);

end

matlabpool close

toc;

% clc;

%======================================

createTable.m

function T = createTable(k, x)

% M = size(x,1)+17; % length of second hashTable (hashTable2)

M = 587474;

d = size(x, 2);

select_d = unidrnd(d, 1, k);

I.d = select_d;

% I.threshold = unifrnd(0, 1, 1, k) * 255; % value interval [0 255]

I.k = k;

T.I = I;

T.randDigits = unidrnd(M, 1, k);

T.buckets = [];

T.index = {};

T.hashTable2 = cell(M,1);

insert_data.m

function T = insert_data(T, x)

% M = size(x, 1) + 17;

M = 587474;

% buck01 = x(T.I.d, :)‘ < repmat(T.I.threshold, size(x,2), 1);

buck01 = x(:, T.I.d);

[uBuck id1 id2] = unique(buck01,‘rows‘);

T.buckets = logical(uBuck);

T.bucket_cnt = length(id1);

key = mod(sum(bsxfun(@times, uBuck, T.randDigits),2), M) + 1; % matalb 下标从 1 开始

T.index = cell(length(id1), 1);

for bb = 1 : length(id1)

sameBucket = find(id2 == bb);

T.index{bb} = [T.index{bb}; sameBucket‘];

T.hashTable2{key(bb)} = [T.hashTable2{key(bb)} bb];

end

Demo_computeError.m (测试精确度)

% clear all; clc;

p = 1;

load([‘..\Data\3_15_Tables.mat‘]);

load(‘..\Data\data.mat‘);

load(‘..\Data\query.mat‘);

Data = Data‘;

avgErr = zeros(1, 50);

MissSum = zeros(1, 50);

picErr = zeros(1,50); picErr(1) = 0.982;

for K = 1 : 10

MissCnt = 0;

ratioCnt = 0;

fid = fopen([‘K_‘ num2str(K) ‘.txt‘], ‘w‘);

for i = 1 : length(Q) % the ith input.

% fprintf(fid, ‘%-5d‘, i);

% fprintf([num2str(i) ‘ ‘]);

q = Q(i,:)‘;

[Id1, Mis] = LSH_Search(q‘, K, Tables, Data, p);

if Mis

MissCnt = MissCnt + 1;

fprintf(fid, ‘Miss\n‘);

continue;

end

% num = num + 1;

% imwrite(uint8(input),[‘.\Data\‘,‘query_subset2\‘,num2str(num),‘.bmp‘]);

Id2 = Linear_Search(q, K, Data, p);

Dlsh = lp_norm(q, Data(:, Id1), p);

Dcst = lp_norm(q, Data(:, Id2), p);

id = Dlsh == Dcst;

ratio = Dcst ./ (Dlsh + 0.00002);

ratio(id) = 1;

% for j = 1 : K

% fprintf(fid, ‘%-8.2f‘, ratio(j));

% end

fprintf(fid, ‘error: %8.3f\n‘, ratio(K));

ratioCnt = ratioCnt + ratio(K);

end

fprintf(fid, ‘Hit times: %-5d Avg. error: %.3f\n‘, length(Q)-MissCnt, ratioCnt/(length(Q)-MissCnt));

avgErr(K) = ratioCnt/(length(Q)-MissCnt);

fprintf(fid, ‘Miss times: %-5d Miss ratio: %.3f‘, MissCnt, MissCnt/length(Q));

MissSum(K) = MissCnt;

picErr(K) = ratioCnt / (length(Q)-MissCnt);

fclose(fid);

end

% figure,plot(1:50,avgErr,‘-r.‘,‘MarkerFaceColor‘,‘g‘),

% xlabel(‘number of K‘),ylabel(‘Error‘),title(‘Error of LSH‘);

% figure, plot(1:50,MissSum/146,‘-r.‘,‘MarkerEdgeColor‘,‘k‘),

% xlabel(‘number of K‘),ylabel(‘Miss ratio‘),title(‘Qurey miss of LSH‘);

figure,plot(1:10,picErr(1:10),‘-r.‘,‘MarkerFaceColor‘,‘g‘),

xlabel(‘number of K‘),ylabel(‘Error‘),title(‘Error of LSH‘);

Linear_Search.m

function Index = Linear_Search(q, K, DataSet, p)

D = feval(‘lp_norm‘, q, DataSet, p);

[~, id] = sort(D);

Index = id(1 : K);

lp_norm.m  (此处使用了 Hamming distance， p = 2 时，可以调整为 l2 范式欧式距离)

%************ get the distance **********************

function distance = lp_norm(x0, x, p)

tem = repmat(x0, 1, size(x,2));

distance = sum((abs(tem - x) .^ p), 1) .^ (1/p);

lookup.m (LSH 查找)

function Index = lookup(T, q) % x can be removed

%========================================================

%************ 参数解释: ***********************

%************ T : 哈希表 ***********************

%************ x : 总数据集 **********************

%************ q: 查询

%

%========================================================

%========================================================

index = [];

% 可并行

for i = 1 : length(T)

tem = getIndex(T(i), q);

index = [index tem];

end

Index = unique(index);

getIndex.m

function tableiIndex = getIndex(T, x0)

M = length(T.hashTable2);

tableiIndex = [];

seq01_x = x0(:, T.I.d);

index_x = mod(sum(bsxfun(@times, seq01_x, T.randDigits),2), M) + 1;

if ~isempty(T.hashTable2{index_x})

index_bucket = T.hashTable2{index_x};

%******************************************

%for i = 1 : length(index_bucket)

%******************************************

uni_index_bucket = index_bucket(find(all(bsxfun(@eq, seq01_x, T.buckets(index_bucket, :)), 2)));

for i = 1 : length(uni_index_bucket)

tableiIndex = [tableiIndex T.index{uni_index_bucket(i)}];

end

end

Linear_Search.m (线性查找)

function Index = Linear_Search(q, K, DataSet, p)

D = feval(‘lp_norm‘, q, DataSet, p);

[~, id] = sort(D);

Index = id(1 : K);

(横轴为 K-NN 中 K 的值，纵轴为准确度)

相关截图：

Algorithm proposed from Papers :

(Indyk 1999) similarity search in hish dimensions via hashing.

(Indyk 2005) Locality-sensitive hashing scheme based on p-stable distributions.

原文：http://www.cnblogs.com/liyangguang1988/p/3885397.html