爬虫遍历中图分类法

很多图书馆网站都有较为完整的网页版本《中图分类法》，不同的分类目录都位于不同的链接网页上

为此专门写了一个遍历该种网站的Java程序，我们以“中国图书馆分类法网站”为例

数据库创建表脚本：

CREATE TABLE [zhtclass] (
       [type] [nvarchar] (100) COLLATE Chinese_PRC_CI_AS NULL ,
       [detail] [nvarchar] (400) COLLATE Chinese_PRC_CI_AS NULL
) ON [PRIMARY]
GO

 

Java代码：

import java.net.URL;
import java.net.URLConnection;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.util.Scanner;

public class Exec {
        public static void main(String args[]) {
                PreparedStatement pstm = null;

                // 下面五个参数可以自行根据网站网页格式调整
                String nullFlag = "没有下级分类";
                String firstFlagAlph = "<ul id=\"list\" class=\"cent\" style=\"list-style:none;\"><li>";
                String startFlagAlph = "<span class=\"code\">";
                String endFlagAlph = "</span>";
                String startFlagName = "\">";
                String endFlagName = "</a>";

                try {
                        Class.forName("com.microsoft.jdbc.sqlserver.SQLServerDriver");
                        Connection con = DriverManager
                                        .getConnection(
                                                        "jdbc:microsoft:sqlserver://localhost:1433;DatabaseName=LIB",
                                                        "sa", "");
                        pstm = con.prepareStatement("insert into zhtclass values(?,?)");
                } catch (Exception e) {
                        System.out.println(e.getLocalizedMessage());
                }
                for (int i = 1; i < 45837; i++) {
                        System.out.println(i);
                        try {
                                StringBuffer sb = new StringBuffer();
                                URL url = new URL("http://www.ztflh.com/?c=" + i);
                                URLConnection urlConn = url.openConnection();
                                urlConn.setReadTimeout(10000);
                                urlConn.setConnectTimeout(10000);
                                urlConn.setDoOutput(true);
                                urlConn.connect();
                                Scanner in = new Scanner(urlConn.getInputStream(),"UTF-8");
                                for (int n = 1; in.hasNextLine(); n++)
                                        sb.append(in.nextLine());

                                if (sb.indexOf(nullFlag) < 0) {
                                        int start = sb.indexOf(firstFlagAlph, 0)
                                                        + firstFlagAlph.length();
                                        int end = start;
                                        while (true) {
                                                if (sb.indexOf(startFlagAlph, end) < 0)
                                                        break;
                                                start = sb.indexOf(startFlagAlph, end)
                                                                + startFlagAlph.length();
                                                end = sb.indexOf(endFlagAlph, start);
                                                String alph = sb.substring(start, end).trim();
                                                start = sb.indexOf(startFlagName, end)
                                                                + startFlagName.length();
                                                end = sb.indexOf(endFlagName, start);
                                                String name = sb.substring(start, end).trim();
                                                pstm.setString(1, alph);
                                                pstm.setString(2, name);
                                                pstm.execute();
                                        }
                                }
                                url = null;
                        } catch (Exception e) {
                                i--;
                                System.out.println(e.getLocalizedMessage());
                        }
                        System.gc();
                }
        }
}