在网上看到网页小偷，有偷笑话的有偷天气的。。。最可笑的是GOOGLE上有个人发布了一篇文章，说是js版本的，我打开一看，language=vbscript.......还被很多人引用，一搜网页小偷，都是那一篇文章，真让人无语。更让人无语的就是那个所谓的超级经典的笑话小偷。引用了很多网站上的js，根本看不到js代码。下载下来源代码，更令人吐血，说是用js实现的，结果里面的js都是0KB，就是空文件。
我今天写了个偷新闻的，由于是“偷”所以程序得基于要偷得网站的代码。
简单的说，就是打开网页，看他的源文件。然后使用java的URL类获取网页内容，使用正则表达式过滤想要的内容，我写的是偷新浪新闻的。还有很多地方没有完善，不过基本功能实现了。打开页面，有个按钮，点开始，就会开始下载和过滤。然后将新闻存入数据库中。
以下代码：
callJsp.jsp:
<%@ page language="java" import="java.util.*" pageEncoding="utf-8"%>
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
"http://www.w3.org/TR/html4/loose.dtd ">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=gb2312">
<title>callJsp</title>
</head>
<body>
<form action="CallServlet">
<button type="submit">start</button>
</form>
</body>
</html>
CallServlet.java:
package servlet;
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import bo.GetContent;
public class CallServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
System.out.println("called");
GetContent gc = new GetContent();
gc.getContent("http://news.sina.com.cn ");
gc.savaToDB();
}
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
this.doGet(request, response);
}
}
实体类：
package dto;
public class New {
private String title;
private int id;
public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
private String content;
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
public String getContent() {
return content;
}
public void setContent(String content) {
this.content = content;
}
}
逻辑类（关键代码）：
package bo;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import dao.NewsDAO;
import dto.New;
public class GetContent {
ArrayList<String> newsList;
Pattern newsPat;
Pattern mainPat;
List<New> news;
NewsDAO newsDAO = new NewsDAO();
int i = 1;
public GetContent() {
String newsReg = "<li(.*)<a(.*)(http://news[/.]sina[/.]com[/.]cn(.*)[/.]shtml)+(.*)>.*</a></li >";
String mainReg = "<p>(.*)</p>";
newsList = new ArrayList<String>();
newsPat = Pattern.compile(newsReg);
mainPat = Pattern.compile(mainReg);
news = new ArrayList<New>();
}
public void getContent(String URLSTR) {
try {
URL url = new URL(URLSTR);
InputStream o = url.openStream();
BufferedReader br = new BufferedReader(new InputStreamReader(o,
"gbk"));
String line = null;
while ((line = br.readLine()) != null) {
Matcher m = newsPat.matcher(line);
if (m.matches()) {
newsList.add(line);
System.out.println(line);
getMain(line);
}
}
} catch (IOException e) {
e.printStackTrace();
}
}
public void getMain(String line) throws MalformedURLException {
String newsContent = null;
int indexOfHrefBegin = line.indexOf("http");
int indexOfHrefEnd = line.indexOf("shtml");
int indexOfTitleBegin = line.indexOf(">", indexOfHrefBegin);
int indexOfTitleEnd = line.indexOf("</a>");
String title = line.substring(indexOfTitleBegin + 1, indexOfTitleEnd);
System.out.println(title);
String hrefStr = line.substring(indexOfHrefBegin, indexOfHrefEnd + 5);
URL u = new URL(hrefStr);
try {
InputStream o = u.openStream();
BufferedReader br = new BufferedReader(new InputStreamReader(o,
"gbk"));
String content = null;
StringBuffer sf = new StringBuffer();
while ((content = br.readLine()) != null) {
Matcher m = mainPat.matcher(content);
if (m.matches()) {
sf.append(content);
newsContent = sf.toString();
}
}
if (sf != null) {
New n = new New();
n.setId(i);
i++;
n.setContent(newsContent);
n.setTitle(title);
news.add(n);
}
} catch (IOException e) {
e.printStackTrace();
}
}
public void savaToDB() {
newsDAO.saveNews(news);
}
}
持久类：
package dao;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
import dto.New;
public class NewsDAO {
Connection con = DBConnection.getConnection();
PreparedStatement pst = null;
ResultSet rs = null;
public void saveNews(List<New> news) {
try {
String sql = "insert into news values(?,?,?)";
pst = con.prepareStatement(sql);
for (New n : news) {
pst.setInt(1, n.getId());
pst.setString(2, n.getTitle());
pst.setString(3, n.getContent());
pst.executeUpdate();
}
} catch (SQLException e) {
e.printStackTrace();
} finally {
try {
pst.close();
con.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
}
数据库链接类：
package dao;
import java.sql.Connection;
import java.sql.DriverManager;
public class DBConnection {
private static final String DBDRIVER = "com.mysql.jdbc.Driver";
private static final String DBURL = "jdbc:mysql://localhost:3306/newsstealing?useUnicode=true&characterEncoding=GBK";
private static final String DBUSER = "root";//用户名
private static final String PASSWORD = "pass";//密码
private static Connection connection = null;
static{
try{
Class.forName(DBDRIVER);
}catch(ClassNotFoundException e){
e.printStackTrace();
}
}
public static Connection getConnection(){
try{
connection = DriverManager.getConnection(DBURL,DBUSER,PASSWORD);
}catch(Exception e){
e.printStackTrace();
}
return connection;
}
}
所有的类都在这里了。根据类名和包名，自己构建一个web project。把mysql的驱动程序放到lib下面。在数据库中创建个数据库，和表（数据库链接类中和dao中可以找到名字）。可以根据自己的条件使用不同的数据库。因为功能还不完善，所有创建表的时候，除了ID，其它的要允许为空值，否则插入的时候有的新闻内容没有获取到，就会出错。
4976
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
