public class WebCrawler { public static void
main(String[] agrs) throws HttpException, IOException{ //要访问的网页URL
String url="http://caikan.com//platform//index.do"; HttpClient
httpClient=new HttpClient(); HttpMethod method=new GetMethod(url);
httpClient.executeMethod(method); String response=null;
if(method.getStatusCode()==HttpStatus.SC_OK){
response=method.getResponseBodyAsString(); } //释放httpClient连接
method.releaseConnection(); //定义正则表达式:获取图片,()表示分组 String
regex="[\\w\\W]*?";
//将返回的网页内容存储成一个 文本文件保存在本地文件夹filetext中,文件名为通过url截取 String
name=url.substring(url.lastIndexOf("/")); File file=new
File("D:\\filetext\\" name ".txt"); FileWriter os=new
FileWriter(file); BufferedWriter bw=new BufferedWriter(os);
//将response写入文件file中 bw.write(response); //从刚保存的文件中获取response内容
FileReader fr=new FileReader(file); BufferedReader br=new
BufferedReader(fr); //当前行的内容 String lineStr=""; //逐行进行判断
while((lineStr=br.readLine())!= null){ Pattern
p=Pattern.compile(regex); Matcher m=p.matcher(lineStr);
if(m.find()){ String src=m.group(1); //判断src的格式的正则表达式 String
regex1="http://[\\W\\w]*?"; Pattern p1=Pattern.compile(regex1);
Matcher m1=p1.matcher(src); //url2作用是特定的图片地址,直接获取 URL url2=null;
if(m1.matches()){ url2=new URL(src); }else{ String
basepath=url.substring(0, url.lastIndexOf("/")); url2=new
URL(basepath "//" src); } //根据src截取图片文件名 String
filename=src.substring(src.lastIndexOf("/") 1); //将图片保存到本地文件中
BufferedImage currentImage=javax.imageio.ImageIO.read(url2);
javax.imageio.ImageIO.write(currentImage, "jpg", new
File("D:\\webImage\\" filename)); } } //关掉所有流 fr.close();
br.close(); os.close(); System.out.println("执行完毕"); }}