网络文件抓取

1、java远程文件抓取，实现远程文件提取，静态生成。

package cn.com;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;

public class Test {

	public static void main(String[] args) throws Exception {
		URL url= new URL("http://www.sina.com"); 
           HttpURLConnection connection =   
            (HttpURLConnection) url.openConnection();   
		connection.setRequestMethod("GET");  
		connection.setRequestProperty("Accept-Language" , "zh-cn");   
		connection.setRequestProperty("Accept-Charset" , "utf-8"); 
		connection.setDoOutput(true);   
		connection.setRequestProperty("User-Agent", "directclient"); 

		String filename = url.getFile();	
		if(!filename.equals("")){
        filename = filename.replace("/", "").substring(0,filename.replace("/", "").indexOf("."));
		}else{
		filename="index";	
		}

        File file = new File("/html/"+filename+".html"); //存放在工作间本地磁盘根目录如：F://html/index.html
        if(file.exists()){    
        	file.delete();    
        }else{   
            try {   
            	file.createNewFile();   
          } catch (IOException e) {   
              e.printStackTrace(); 
          }   
        }   
        connection.connect();
        String charset = connection.getContentType();
        if(charset.length()>9 && !charset.contains("none")){
        	charset = charset.substring(charset.indexOf("=")+1,charset.length());
        }else{
        	charset = "utf-8";
        }
        InputStream is = connection.getInputStream();          
        BufferedReader br = new BufferedReader(new InputStreamReader(is,charset)); 
        FileWriter out = new FileWriter("/html/"+filename+".html");   
        String strLine = null;   
        while((strLine = br.readLine()) != null){  
           System.out.println(strLine);   
           out.write(strLine+"\n");   
           out.flush(); 
        }  

        out.close(); 
        br.close();   
        connection.disconnect();   
        System.out.println("静态页面生成成功！");
	}
}

2、php远程提取超链接，

<?php
$str = file_get_contents("http://www.sina.com");
$pat = '/<a(.*?)href="(.*?)"(.*?)>(.*?)<\/a>/i';      

preg_match_all($pat, $str, $m);     

foreach ($m as $value){
	foreach ($value as $val) {
		echo $val."<br>";
	}
}
?>

3、java 内容搜索

package cn.com;

import java.io.File;
import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.List;

public class Test {

	public static void main(String[] args) {
		getText(new File("D:/Program Files/Apache Software Foundation/Apache2.2/htdocs/tun2/b2b"));
		System.out.println("ok");
	}

	 public static void getText(File f){  
		         File file[] = f.listFiles();  
		         for(File dir : file){    
		             if(dir.isDirectory()){  
		                 getText(dir);  
		             }else{  		                
		                     try {  
		                         RandomAccessFile raf = new RandomAccessFile(dir, "r");  
		                         String strLine = null;  
		                         while((strLine=raf.readLine())!=null){  
		                             if((new String(strLine.getBytes("ISO-8859-1"),"utf8")).contains("show_bd_pics")){
		                            	System.out.println(dir);
		                            	break;
		                             }
		                         }  
		                     } catch (Exception e) {  
		                         e.printStackTrace();  
		                     }  
		             }  

		         }  
		      }  
}