网络爬虫是搜索引擎的一个重要的部分。爬虫的根本原理就是下载页面，然后进行解析。Web上的存储着海量数据，怎么样才能将海量数据尽快的下载到本机上？这是网络爬虫设计的一个方案。采取多线程技术。以下代码实现了将网页的数据存储到XML文档。希望能提出更好的方案。
- import java.awt.*;
- import java.awt.event.*;
- import javax.swing.*;
- import java.util.*;
- import java.io.*;
- import com.heaton.bot.*;
- import org.w3c.dom.*;
- import org.cyberneko.html.parsers.*;
- import org.xml.sax.*;
- import org.apache.html.dom.*;
- import javax.xml.parsers.*;
- import javax.xml.transform.*;
- import javax.xml.transform.dom.*;
- import javax.xml.transform.stream.*;
- /**
- * 网络爬虫，通过深度优先算法将互联网上的网页下载解析。经过测试，能正常运行！
- */
- public class mySpider extends JFrame implements ISpiderReportable {
- /**
- * @param args
- */
- Spider _Spider = null;
- Document doc = null;
- int _pagesCount;
- public mySpider(){
- setTitle("网络爬虫之Spider");
- getContentPane().setLayout(null);
- setSize(405,268);
- setVisible(false);
- D.setHorizontalTextPosition(
- SwingConstants.LEFT);
- D.setVerticalTextPosition(
- SwingConstants.TOP);
- D.setVerticalAlignment(
- SwingConstants.TOP);
- D.setText("下载的页面数目： ");
- getContentPane().add(D);
- D.setBounds(12,12,384,24);
- JLabel2.setText("URL:");
- getContentPane().add(JLabel2);
- JLabel2.setBounds(12,36,36,24);
- getContentPane().add(_url);
- _url.setBounds(48,36,348,24);
- JLabel3.setText("选择存储XML文档的目录：");
- getContentPane().add(JLabel3);
- JLabel3.setBounds(12,72,384,24);
- getContentPane().add(_save);
- _save.setBounds(12,96,384,24);
- _go.setText("GO!");
- getContentPane().add(_go);
- _go.setBounds(96,228,216,24);
- getContentPane().add(_current);
- _current.setBounds(12,204,384,12);
- JLabel4.setText("Number of pages:");
- getContentPane().add(JLabel4);
- JLabel4.setBounds(12,180,120,12);
- _pages.setText("0");
- getContentPane().add(_pages);
- _pages.setBounds(120,180,108,12);
- JLabel6.setText("选择Log，当前需要保存的日志:");
- getContentPane().add(JLabel6);
- JLabel6.setBounds(12,120,384,24);
- _logPath.setText("./Spider.log");
- getContentPane().add(_logPath);
- _logPath.setBounds(12,144,384,24);
- _go.setActionCommand("jbutton");
- SymAction lSymAction = new SymAction();
- _go.addActionListener(lSymAction);
- SymWindow aSymWindow = new SymWindow();
- this.addWindowListener(aSymWindow);
- try{
- DOMImplementation domImpl = DocumentBuilderFactory
- .newInstance().newDocumentBuilder().getDOMImplementation();
- doc = domImpl.createDocument(null,"spider",null);
- }catch(ParserConfigurationException e){
- e.printStackTrace();
- }catch(DOMException e){
- e.printStackTrace();
- }
- }
- public void setVisible(boolean b){
- if(b)
- setLocation(50,50);
- super.setVisible(b);
- }
- public static void main(String[] args) {
- // TODO Auto-generated method stub
- (new mySpider()).setVisible(true);
- }
- public void addNotify(){
- Dimension size = getSize();
- super.addNotify();
- if(frameSizeAdjusted)
- return;
- frameSizeAdjusted = true;
- Insets insets = getInsets();
- JMenuBar menuBar = getRootPane().getJMenuBar();
- int menuBarHeight = 0;
- if(menuBar!=null)
- menuBarHeight = menuBar.getPreferredSize().height;
- setSize(insets.left+
- insets.right+
- size.width,
- insets.top+
- insets.bottom+
- size.height+menuBarHeight);
- }
- boolean frameSizeAdjusted = false;
- JLabel D = new JLabel();
- JLabel JLabel2 = new JLabel();
- JTextField _url = new JTextField();
- JLabel JLabel3 = new JLabel();
- JTextField _save = new JTextField();
- JButton _go = new JButton();
- JLabel _current = new JLabel();
- JLabel JLabel4 = new JLabel();
- JLabel _pages = new JLabel();
- JLabel JLabel6 = new JLabel();
- JTextField _logPath = new JTextField();
- class SymAction implements ActionListener{
- public void actionPerformed(ActionEvent event){
- Object object = event.getSource();
- if(object == _go)
- Go_actionPerformed(event);
- }
- }
- protected void processFile(HTTP file){
- try{
- if(_save.getText().length()>0){
- int i=file.getURL().lastIndexOf('/');
- if(i!=-1){
- int iPoint = file.getURL().lastIndexOf('.');
- String extendName = file.getURL().substring(iPoint+1);
- if(extendName.equals("html") || extendName.equals("htm") || extendName.equals("shtml")){
- String fileBody = new String(file.getBody().getBytes("iso-8859-1"),"GBK");
- DOMFragmentParser parser = new DOMFragmentParser();
- DocumentFragment node =
- new HTMLDocumentImpl().createDocumentFragment();
- try {
- parser.setProperty("http://cyberneko.org/html/properties/default-encoding","GBK");
- parser.parse(new InputSource(new ByteArrayInputStream(fileBody.getBytes())), node);
- }catch (IOException e) {
- e.printStackTrace();
- }catch (SAXException e) {
- e.printStackTrace();
- }
- StringBuffer sb = new StringBuffer();
- getText(sb, node, "title");
- String title = sb.toString();
- sb.setLength(0);
- getText(sb, node,"body");
- String text = sb.toString();
- text = text.replaceAll("<","<")
- .replaceAll(">",">");
- if(title.length()!=0 && text.length()!=0)
- addElementNode(doc,title,text,file.getURL());
- }
- }
- }
- }catch(Exception e){
- Log.logException("Can't save output file: ",e);
- }
- }
- private Element createTitleElement(Document docs,String title){
- Element titleElement = docs.createElement("TITLE");
- titleElement.setTextContent(title);
- return titleElement;
- }
- private Element createBodyElement(Document docs,String body){
- Element bodyElement = docs.createElement("BODY");
- bodyElement.setTextContent(body);
- return bodyElement;
- }
- private Element createURLElement(Document docs,String URL){
- Element URLElement = docs.createElement("URL");
- URLElement.setTextContent(URL);
- return URLElement;
- }
- public void addElementNode(Document docs,String title,String body,String URL){
- Element HTMLElement = docs.createElement("HTMLPAPER");
- HTMLElement.appendChild(createTitleElement(docs,title));
- HTMLElement.appendChild(createBodyElement(docs,body));
- HTMLElement.appendChild(createURLElement(docs,URL));
- docs.getDocumentElement().appendChild(HTMLElement);
- }
- private void getText(StringBuffer sb, Node node) {
- if (node.getNodeType() == Node.TEXT_NODE) {
- sb.append(node.getNodeValue());
- }
- NodeList children = node.getChildNodes();
- if (children != null) {
- int len = children.getLength();
- for (int i = 0; i < len; i++) {
- getText(sb, children.item(i));
- }
- }
- }
- private boolean getText(StringBuffer sb, Node node,
- String element) {
- if (node.getNodeType() == Node.ELEMENT_NODE) {
- if (element.equalsIgnoreCase(node.getNodeName())) {
- getText(sb, node);
- }
- }
- NodeList children = node.getChildNodes();
- if (children != null) {
- int len = children.getLength();
- for (int i = 0; i < len; i++) {
- if (getText(sb, children.item(i), element)) {
- return true;
- }
- }
- }
- return false;
- }
- void Go_actionPerformed(ActionEvent event){
- IWorkloadStorable wl = new SpiderInternalWorkload();
- if(_Spider!=null){
- Runnable doLater = new Runnable(){
- public void run(){
- _go.setText("Canceling...");
- }
- };
- SwingUtilities.invokeLater(doLater);
- _Spider.halt();
- return;
- }
- try{
- if(_url.getText().length()>0){
- HTTPSocket http = new HTTPSocket();
- http.send(_url.getText(),null);
- }else{
- _current.setText("<<distributed mode>>");
- }
- }catch(Exception e){
- JOptionPane.showMessageDialog(this,e,"Error",JOptionPane.OK_CANCEL_OPTION,null);
- return;
- }
- Runnable doLater = new Runnable(){
- public void run(){
- _go.setText("Cancel");
- _current.setText("Loading...");
- }
- };
- SwingUtilities.invokeLater(doLater);
- _pagesCount = 0;
- if(_logPath.getText().length()>0){
- File file = new File(_logPath.getText());
- file.delete();
- Log.setLevel(Log.LOG_LEVEL_NORMAL);
- Log.setFile(true);
- Log.setConsole(false);
- Log.setPath(_logPath.getText());
- }
- try{
- wl = new SpiderSQLWorkload(
- "sun.jdbc.odbc.JdbcOdbcDriver",
- "jdbc:odbc:WORKLOAD");
- }catch(Exception e){
- JOptionPane.showMessageDialog(this,
- e,
- "Error",
- JOptionPane.OK_CANCEL_OPTION,
- null);
- }
- _Spider = new Spider(this,
- _url.getText(),
- new HTTPSocket(),
- 100,
- wl);
- _Spider.setMaxBody(200);
- _Spider.start();
- }
- public boolean foundInternalLink(String url){
- return true;
- }
- public boolean foundExternalLink(String url){
- return false;
- }
- public boolean foundOtherLink(String url){
- return false;
- }
- class UpdateTarget implements Runnable{
- public String _t;
- public void run(){
- _current.setText(_t);
- _pages.setText(""+_pagesCount);
- }
- }
- public void processPage(HTTP page){
- _pagesCount++;
- UpdateTarget ut = new UpdateTarget();
- ut._t = page.getURL();
- SwingUtilities.invokeLater(ut);
- processFile(page);
- }
- public void completePage(HTTP page, boolean error){
- }
- public boolean getRemoveQuery(){
- return true;
- }
- public void spiderComplete(){
- if(_Spider.isHalted()){
- JOptionPane.showMessageDialog(this,
- "下载正在被取消...... "+
- "请检查日志的错误！.",
- "完成下载！",
- JOptionPane.OK_CANCEL_OPTION,
- null);
- }else{
- JOptionPane.showMessageDialog(this,
- "下载完成..... "+
- "请检查日志的错误！.",
- "完成下载！",
- JOptionPane.OK_CANCEL_OPTION,
- null);
- }
- DOMSource doms = new DOMSource(doc);
- File f = new File(_save.getText(),"HTMLPraser.xml");
- StreamResult sr = new StreamResult(f);
- try{
- TransformerFactory tf = TransformerFactory.newInstance();
- Transformer t = tf.newTransformer();
- Properties properties = t.getOutputProperties();
- properties.setProperty(OutputKeys.ENCODING,"GBK");
- properties.setProperty(OutputKeys.INDENT,"yes");
- t.setOutputProperties(properties);
- t.transform(doms,sr);
- }catch(TransformerConfigurationException tce){
- tce.printStackTrace();
- }catch(TransformerException te){
- System.out.println("转换错误..../n-------");
- te.printStackTrace();
- }
- _Spider = null;
- Runnable doLater = new Runnable(){
- public void run(){
- _go.setText("GO!!");
- }
- };
- SwingUtilities.invokeLater(doLater);
- }
- class SymWindow extends WindowAdapter{
- public void windowClosed(WindowEvent event){
- Object object = event.getSource();
- if(object==mySpider.this)
- GetSite_windowClosed(event);
- }
- }
- void GetSite_windowClosed(WindowEvent event){
- System.exit(0);
- }
- }
- <font color="#000000">以上是网络爬虫的代码。需要导入第三方包，可以在Eclipse环境下开发。但是本人</font>
- <font color="#000000">配置环境变量，直接在记事本中开发。第三方包可以在网上下载。也可以向我索取。</font>
- <font color="#000000">留下个人E-mail即可。</font>
3464
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
