lucene+nutch学习笔记之二：搜索引擎原理

       整个互联网可以看成是一个蜘蛛网，相互关联，可以感觉冒一个线索，找到其他的分支。搜索引擎主要包括这几个步骤：

 

  1爬虫抓取互联网的网页。

  2对网页内容进行分析

  3对分析后的内容建立索引

  4网页检索结果排序

  5提供接口来交互。

1网页html源码的获取(WebHttpClient) 写道

package com.lucene.downpage;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.net.Socket;
import java.net.UnknownHostException;

public class WebHttpClient {

/**
* @param args
*/
public static void main(String[] args) {
try {
Socket webClient=new Socket("www.bnu.edu.cn",80);
PrintWriter result=new PrintWriter(webClient.getOutputStream(),true);
BufferedReader buff=new BufferedReader(new InputStreamReader(webClient.getInputStream()));
result.println("GET / HTTP/1.1");//注意这里的格式，空格一定要写对，不然找不出来。。。。
result.println("Host:bnu.edu.cn");
result.println("Connection:Close");
result.println();
boolean bRet=true;
StringBuffer sb=new StringBuffer(8096);
while(bRet){
if(buff.ready()){
int idx=0;
while(idx!=-1){
idx=buff.read();
sb.append((char)idx);
}
bRet=false;
}
}
System.out.println(sb.toString());
webClient.close();
} catch (UnknownHostException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
System.err.println("下载失败。。。请检查输入的地址是否正确");
System.exit(1);
}
}

}

 

2保存到本地为HTML文件 写道

package com.lucene.downpage;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.net.Socket;
import java.net.UnknownHostException;

public class WebCrawler {

/**
* @param args
*/
private static String Text_File_path = "D:\\workshop\\ch2\\htmlsrc.html";

public static void main(String[] args) {
try {
File file = new File(Text_File_path);
FileWriter fp = new FileWriter(file);
/* 生成下载对象 */
Socket webClient = new Socket("www.bnu.edu.cn", 80);
PrintWriter result = new PrintWriter(webClient.getOutputStream(),
true);
BufferedReader buff = new BufferedReader(new InputStreamReader(
webClient.getInputStream()));
/* 发送Http Request请求 */
result.println("GET / HTTP/1.1");// 注意这里的格式，空格一定要写对，不然找不出来。。。。
result.println("Host:bnu.edu.cn");
result.println("Connection:Close");
result.println();
boolean bRet = true;
StringBuffer sb = new StringBuffer(8096);
while (bRet) {
if (buff.ready()) {
int idx = 0;
while (idx != -1) {
idx = buff.read();
if (idx == '<') {
break;
}

}
while (idx != -1) {
sb.append((char) idx);
idx = buff.read();
}
bRet = false;
}
}
System.out.println(sb.toString());
fp.write(sb.toString());
webClient.close();
fp.close();
} catch (UnknownHostException e) {
// TODO Auto-generated catch block
System.err.println("无法访问主机");
System.exit(1);
} catch (IOException e) {
// TODO Auto-generated catch block
System.err.println("下载失败。。。请检查输入的地址是否正确");
System.exit(1);
}
}

}

 

分析本地网页内容，提取文本 写道

package com.lucene.downpage;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;

public class WebParser {

/**
* @param args
*/
private static String src_File_Path="D:\\workshop\\ch2\\htmlsrc.html";
private static String dst_File_Path="D:\\workshop\\ch2\\puresrc.txt";
public static void main(String[] args) throws IOException {
Parser();

}
public static void Parser() throws IOException{
boolean bContent=true;
StringBuffer st=new StringBuffer(8096*2);
char[] cBuff=new char[8096*2];
int nCount=0;
File srcFile=new File(src_File_Path);
try {
FileReader fpReader=new FileReader(srcFile);
File dstFile=new File(dst_File_Path);
FileWriter fpWriter=new FileWriter(dstFile);
nCount=fpReader.read(cBuff);
for(int i=0;i<nCount;i++){
if(bContent==false){
if(cBuff[i]=='>'){
bContent=true;
}else{
continue;
}
}
else{
if(cBuff[i]=='<'){
bContent=false;
continue;
}else if(cBuff[i]=='\n'||cBuff[i]==' '||cBuff[i]==' ' || cBuff[i]==' '){
continue;
}else if(cBuff[i]=='&'&& cBuff[i+1]=='n'&& cBuff[i+2]=='b' && cBuff[i+3]=='s' && cBuff[i+4]=='p'&& cBuff[i+5]==';'){
i=i+5;
continue;
}
st.append(cBuff[i]);
fpWriter.write(cBuff[i]);
}

}

System.out.println(st.toString());
fpReader.close();
fpWriter.close();
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}


}


}

 

对提取的文本进行分词过滤 写道

package com.lucene.downpage;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;

public class WebParseFilter {

/**
* @param args
*/
private static String src_File_Path="D:\\workshop\\ch2\\htmlsrc.html";
private static String dst_File_Path="D:\\workshop\\ch2\\puresrc.txt";
public static void main(String[] args) {
try {
ParseFilter();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public static void ParseFilter() throws IOException{
int j=0;
boolean flag=true;
boolean bContent=true;
StringBuffer sBuffer=new StringBuffer(8096*2);
char[] cBuffer=new char[8096*2];
char[] dstBuffer=new char[8096*2];
int nCount=0;
File srcFile=new File(src_File_Path);
try {
FileReader fpReader=new FileReader(srcFile);
File dstFile=new File(dst_File_Path);
FileWriter fw=new FileWriter(dstFile);
nCount=fpReader.read(cBuffer);
for(int i=0;i<nCount;i++){
if(bContent==false){
if(cBuffer[i]=='>'){
bContent=true;
}else{
continue;
}
}else{
if(cBuffer[i]=='<'){
bContent=false;
continue;
}else if(cBuffer[i]=='\n'||cBuffer[i]==' '){
continue;
}else if(cBuffer[i]=='&'||cBuffer[i+1]=='n'||cBuffer[i+2]=='b'||cBuffer[i+3]=='s'||cBuffer[i+4]=='p'||cBuffer[i+5]==';'){
i=i+5;
continue;

}
dstBuffer[j++]=cBuffer[i];
}
}
flag=true;
for(int m=0;m<j;m++){
if((dstBuffer[m]<='Z'&&dstBuffer[m]>='A')||(dstBuffer[m]<='Z'&& dstBuffer[m]>='a')||(dstBuffer[m]<='9'&&dstBuffer[m]>='0')){
if(flag==false){
sBuffer.append(' ');
}
sBuffer.append(dstBuffer[m]);
flag=true;
}else{
if(dstBuffer[m]=='.'||dstBuffer[m]=='|'||dstBuffer[m]=='"'||dstBuffer[m]==':'||dstBuffer[m]==';'){
sBuffer.append(' ');
continue;
}
if(flag==true){
sBuffer.append(' ');
}
sBuffer.append(dstBuffer[m]);
sBuffer.append(' ');
flag=false;
}

}
System.out.println(sBuffer.toString());
fw.write(sBuffer.toString());
fpReader.close();
fw.close();
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}


}
}

 

对分词后的文本建立索引 写道

package com.lucene.downpage;

public class InfoItem {

/**
* @param args
*/
public int fileId;
public int offset;
InfoItem next;
public InfoItem(){
fileId=0;
offset=0;
next=null;

}
public InfoItem(int id,int pos){
fileId=id;
offset=pos;
next=null;

}
public InfoItem getNext() {
return next;
}
public void setNext(InfoItem next) {
this.next = next;
}
public int getFileId() {
return fileId;
}
public int getOffset() {
return offset;
}
}

 

 

建立索引并查询 写道

package com.lucene.downpage;

import java.util.Hashtable;

public class WordIndex {

/**
* @param args
*/
static Hashtable keyWordIdx;
static String[] FileList = {"北 京 师 范 大 学", "北 师 大 附 属 实 验 小 学",
"北 师 大 第 二 附 属 中 学" };

public static void main(String[] args) {
try {
index();
search("北");
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public static void index() throws Exception{
InfoItem item,item2;
System.out.println("index:==========begin=================");
keyWordIdx=new Hashtable();
try{
System.out.println("index:Hash Table initial Size:"+keyWordIdx.size());
for(int i=0;i<3;i++){
int len=FileList[i].length();
for(int j=0;j<len;j++){
item=new InfoItem(i,j);
String key=FileList[i].substring(j,j+1);
System.out.print(key);
if(!keyWordIdx.containsKey(key)){
keyWordIdx.put(key,item);

}else{
item2=(InfoItem)keyWordIdx.get(key);
item.setNext(item2);
keyWordIdx.put(key, item);
}
}
System.out.println("");
}
System.out.println("index:Hash Table finish Size:"+keyWordIdx.size());
}catch(Exception ex){
throw ex;
}
System.out.println("====================end==================");
}

public static void search(String keyWord) throws Exception{
InfoItem item;
System.out.println("search:=============begin================");
if(null==keyWordIdx){
return ;
}
try{
item=(InfoItem)keyWordIdx.get(keyWord);
while(item!=null){
System.out.println("Search:File number:"+item.getFileId());
System.out.println("Search:File offset:"+item.getOffset());
System.out.println("Search:File Content:"+FileList[item.getOffset()]);
item=item.getNext();
}
System.out.println("search:============end================");

}catch(Exception e){
throw e;
}


}

}