webcollector mysql_如何使用webcollector爬取搜索引擎

展开全部

使用WebCollector来爬取百度搜索引擎按照关键字搜索的结果页面，解析规则可能会随百度搜索的改版而失效e68a84e8a2ad62616964757a686964616f31333363373766。

代码如下：

[java] view plain copy

package com.wjd.baidukey.crawler;

import java.io.ByteArrayInputStream;

import java.io.FileOutputStream;

import java.io.IOException;

import java.net.URLEncoder;

import java.sql.Connection;

import java.sql.DriverManager;

import java.sql.PreparedStatement;

import java.sql.ResultSet;

import java.sql.SQLException;

import java.sql.Statement;

import java.text.SimpleDateFormat;

import java.util.Date;

import java.util.HashMap;

import java.util.TimeZone;

import org.apache.poi.poifs.filesystem.DirectoryEntry;

import org.apache.poi.poifs.filesystem.POIFSFileSystem;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

import cn.edu.hfut.dmic.contentextractor.ContentExtractor;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;

import cn.edu.hfut.dmic.webcollector.model.Page;

import cn.edu.hfut.dmic.webcollector.plugin.ram.RamCrawler;

public class BdiduKeywordCrawler extends RamCrawler{

private Connection connection;

private PreparedStatement pstatement;

// 连接MySql数据库，用户名root，密码mahao

String url = "jdbc:mysql://localhost:3306/wjd";

String username = "root";

String password = "mahao";

//保存抽取到的数据

StringBuilder result = new StringBuilder();

public BdiduKeywordCrawler(String keyword, int maxPageNum) throws Exception {

for (int pageNum = 1; pageNum <= maxPageNum; pageNum++) {

String url = createUrl(keyword, pageNum);

CrawlDa