最近由于公司业务需要,需要从某网站爬取数据,在正则表达式上费了一番功夫,其他地方还算OK,这篇文章从网站爬取数据开始(分页的),到用jdbc插入MySQL数据库,然后从MySQL数据库中导出数据,一条龙!!!话不多说,直接上代码:
package com;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class CrawlInfoUtil {
public static void main(String[] args) throws Exception{
//1、趴取网页上信息,拼装List
for(int i = 1; i < 50; i++){
StringBuffer sb = new StringBuffer("http://www.chinaplasonline.com/ExhibitorList16/lang-simp/page-");
sb.append(i).append("/src-12/s-cps11/od-0/hall-/asc-1/ChemicalZone.aspx");
System.out.println("-----------第"+i+"页开始-----------");
System.out.println("第"+i+"页URL:"+sb.toString());
String content = httpRequest(sb.toString());
List list = htmlFilter(content);
//2、连接数据库
Connection conn = connectDataBase();
//3、将拼装的数据存入数据库中
insertData2DataBase(list, conn);
}
}
private static String httpRequest(String requestUrl) {
StringBuffer buffer = null;
BufferedReader bufferedReader = null;
InputStreamReader inputStreamReader = null;
InputStream inputStream = null;
HttpURLConnection httpUrlConn = null;
try {
// 建立get请求
URL url = new URL(requestUrl);
httpUrlConn = (HttpURLConnection) url.openConnection();
httpUrlConn.setDoInput(true);
httpUrlConn.setRequestMethod("GET");
// 获取输入流
inputStream = httpUrlConn.getInputStream();
inputStreamReader = new InputStreamReader(inputStream, "utf-8");
bufferedReader = new BufferedReader(inputStreamReader);
// 从输入流读取结果
buffer = new StringBuffer();
String str = null;
while ((str = bufferedReader.readLine()) != null) {
buffer.append(str);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
// 释放资源
if (bufferedReader != null) {
try {
bufferedReader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (inputStreamReader != null) {
try {
inputStreamReader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (inputStream != null) {
try {
inputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (httpUrlConn != null) {
httpUrlConn.disconnect();
}
}
return buffer.toString();
}
private static List htmlFilter(String html){
List list = new ArrayList();
System.out.println("----------------开始解析-----------------");
Pattern p = Pattern.compile("()(.*?)()");
Matcher m = p.matcher(html);
int i = 0;
while(m.find()){
EnterprisInfoVo vo = new EnterprisInfoVo();
String innerTR = m.group(2);
//获取展商名称和URL
//链接到新页面的
Pattern tdPattern = Pattern.compile("(.*)(
package com;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class CrawlInfoUtil {
)(.*?)()(.*?)()(.*)");
} 以上是把网页上的数据插入到数据库的过程,这里需要注意一下java的正则表达式,要注意find()方法和matches方法的区别,find()方法是会自动将符合条件的目标移动到下一个,并且是按照部分匹配的,注意我的代码中matches的正则开始和结尾都用到了(.*),而find()方法没有用;matches()方法刚好相反; 下面把数据库中的数据导入到桌面上: package com; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import org.apache.poi.hssf.usermodel.HSSFCell; import org.apache.poi.hssf.usermodel.HSSFCellStyle; import org.apache.poi.hssf.usermodel.HSSFFont; import org.apache.poi.hssf.usermodel.HSSFRow; import org.apache.poi.hssf.usermodel.HSSFSheet; import org.apache.poi.hssf.usermodel.HSSFWorkbook; import org.apache.poi.hssf.util.Region; import com.mysql.jdbc.PreparedStatement; public class ExcelTest { ", "")); } |