java 使用Jsoup 爬取 中国政府采购网数据

int isSize = 1;
Integer pageSize = 0;
Integer timeType = 4; // 时间类型 6: 指定时间 5：近半年 0：今日 4：近3月

public String createhttpClient(Integer id, String kw, String startTime, String endTime) {
logger.info("++++++++++++++++++++++++++++++++开始拉取中国政府采购网数据+++++++++++++++++++++++++++++++++");
long start = System.currentTimeMillis();
Integer totalSize = null;
try {
Integer is_homepage = 0;
k = toUtf8String(kw);
s = URLEncoder.encode(startTime, “UTF-8”);
e = URLEncoder.encode(endTime, “UTF-8”);
pageSize = 1;
ur = “http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=” + pageSize + “&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=0&dbselect=bidx&kw=” + k + “&start_time=” + s + “&end_time=” + e + “&timeType=” + timeType + “&displayZone=&zoneId=&pppStatus=0&agentName=”;
totalSize = fetchHomePageUrl(ur);

    } catch (Exception er) {
        logger.error("========================获取中国政府采购网首页数据失败，重新获取中.....=================================");
        String message = er.getMessage();
        if (message != null) {
            fetchException(message, id, pageSize, ur,0) ;
        }
    }

        if (totalSize>0){
            while (true){
                //遍历每一页的url
               /* for (int h = 1; h <= size; h++) {
                    if (isSize> size){
                        break;
                    }
                    pageSize = h;
                    //  String urlt = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=" + pageSize+ "&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=0&dbselect=bidx&kw=" + k + "&start_time=" + s + "&end_time=" + e + "&timeType=" + timeType + "&displayZone=&zoneId=&pppStatus=0&agentName=";
                    fetchingUrl(id, h, ur);
                    isSize ++ ;
                }*/
                if (isSize> totalSize){
                    break;
                }
                pageSize = isSize;
                fetchingPageUrl(id, pageSize, ur);
                isSize++;
            }
        }else {
            logger.info("关键词："+kw + "没有查询到相关数据-"+new Date());
            return  null;
        }
    return null;
}

// 首次访问时的url ,需查找当前查询词所在的所总页数
public Integer fetchHomePageUrl(String ur){
Integer size = null;
try {
//maxBodySize : 0 不限制大小， 1： 1m userAgent: 伪装成浏览器
Document doc = Jsoup.connect(ur).header(“Accept-Encoding”, “gzip, deflate”)
// .userAgent(“Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0”)
.userAgent(“Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.64 Safari/537.31”)
.maxBodySize(0).timeout(600000).get();
Elements eli = doc.select(".vT-srch-result-list-con2 .vT-srch-result-list .vt-srch-result-list-bid>li");
if (eli.size() > 0) {
//获取分页html
Elements allElements = doc.select(".vT_z .pager>script");
if (allElements.size() > 0) {
String pager = allElements.get(0).childNodes().toString();
String startStr = pager.substring(7, pager.length());
String objectStr = startStr.substring(0, startStr.lastIndexOf(")"));
//string转json对象
JSONObject jsonObject = JSONObject.parseObject(objectStr);
// 共有多少页
size = Integer.valueOf((Integer) jsonObject.get(“size”));
}
}
} catch (IOException e1) {
e1.printStackTrace();
}
return size;
}

// 遍历页数，存入数据库
public void fetchingPageUrl(Integer id, Integer pageSize, String fetchUrl) {
    Integer is_homePage = 1;  // 0 ： 首页  1： 否
    //存govCCgp表
    GovCcgpModel govCcgpModel = new GovCcgpModel();
    GovEnclosureModel enclosureModel = new GovEnclosureModel();
    GovOutlineModel govOutlineModel = new GovOutlineModel();
    GovTextModel govTextModel = new GovTextModel();
    String noticeOutline = null;
    String noticeText = null;
    try {
        Document pDoc = Jsoup.connect(fetchUrl).header("Accept-Encoding", "gzip, deflate")
                // .userAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0")
                .userAgent("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.64 Safari/537.31")
                .maxBodySize(0)
                .timeout(600000).get();
        //根据css样式 获取当前页的所有url
        Elements elements = pDoc.select(".vT_z .vT-srch-result-list-bid>li");
        // Elements hrefs = doc.select("a[href]");
        //遍历当前页的所有url
        for (Element element : elements) {
            List<String> names = new ArrayList<>();
            List<String> enclousureList = new ArrayList<>();
            Random rand = new Random();
            //生成1-15的随机数，睡眠
            Integer r = rand.nextInt(16) + 1;
            Thread.sleep(r);
            Elements childs = element.children();
            //获取a标签的href 网址， select获取到当前a标签attr  href获取到网址;
            String url = childs.select("a").attr("href");
            String str = url.substring(0, 10);
            String govId = url.substring(22, url.length());

            //判断a标签的网址是否为“http://www.”开头
            // if (!str.equals("http://www")) return null;
            Document document = Jsoup.connect(url).timeout(6000).get();
            //取正文标题 div //进入url,将里面的内容处理存入数据库
            // Elements eTitle = document.select(".vF_detail_header>h2");
            // Elements eTime = document.select(".vF_detail_header>p");
            //招标公告名称
            String title = document.title();
            //采购人信息
            String purInformation = childs.get(2).text();
            // 采购人信息 “|” 每一次出现的位置
            int p = purInformation.indexOf("|");
            SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy.MM.dd hh:mm:ss");
            //公告标书时间
            Date createTime = simpleDateFormat.parse(purInformation.substring(0, p));
            //采购人
            String purchaser = purInformation.substring(p + 1, purInformation.length());
            //公告概要  表信息html
            Elements outLine = document.select(".main .main_container .vF_deail_maincontent .vF_detail_main .table>table");
            //解析表信息
            Document outlineDoc = Jsoup.parse(String.valueOf(outLine));
            //遍历每一行表格信息，判断 是否包含附件
            Elements trs = outlineDoc.select("table").select("tr");
            String texTtable = outlineDoc.select("table").html();
            for (int i = 0; i < trs.size(); i++) {
                //判断表格内是否存在附件
                Elements tds = trs.get(i).select("td");
                String enclosureStr = tds.get(0).text();
                if ("附件：".equals(enclosureStr)) {
                    //存入附件表
                    for (int j = i + 1; j < trs.size(); j++) {
                        Elements tdsEnclo = trs.get(j).select("td");
                        String encloDoc = tdsEnclo.get(1).html();
                        //获取a标签的id
                        String uuid = tdsEnclo.get(1).select("a").attr("id");
                        if (uuid != null && uuid.length() > 0) {
                            //组合路径 下载附件
                            String enUrl = "http://www.ccgp.gov.cn/oss/download?uuid=" + uuid;
                            String fileName = tdsEnclo.get(1).text();
                            govCcgpModel.setIsEnclosure(0);
                            enclosureModel.setGovId(govId);
                            enclosureModel.setName(String.valueOf(fileName));
                            //指定下载到本地的路径
                            download(enUrl, fileName, PATH, enclosureModel);
                        }
                    }
                    break;
                }
                //存放在noticeountline 中每行信息以&/n&进行分隔；
                if (noticeOutline != null && noticeOutline.length() != 0) {
                    noticeOutline = noticeOutline + "" + trs.get(i).text() + "&/n&";
                } else {
                    noticeOutline = trs.get(i).text() + "&/n&";
                }
            }
            if (noticeOutline != null && noticeOutline.length() != 0) {
                //存入概要表
                govOutlineModel.setGovId(govId);
                govOutlineModel.setOutlineText(noticeOutline);
                govOutlineModel.setOutlineTable(texTtable);
                govOutlineDao.insertOutLine(govOutlineModel);
            }

            //存govCCgp表
            Integer en_id = govCcgpModel.getIsEnclosure();
            if (en_id == null ) {
                govCcgpModel.setIsEnclosure(1);
            }
            govCcgpModel.setGovId(govId);
            govCcgpModel.setPurchaser(purchaser);
            govCcgpModel.setTitle(title);
            govCcgpModel.setGov_keyword_id(id);
            govCcgpModel.setIs_enable(0);
            govCcgpModel.setSource_site("ccgp.gov.cn");//来源网站 中国政务采购网
            govCcgpModel.setCreateTime(createTime);
            govCcgpModel.setCreateOn(new Date());
            //govId 是否存在， 存在 则更新
            List<HashMap<String, String>> govidList = govCcgpDao.findGovId(govId);
            if (govidList.size() > 0) {
                govCcgpDao.updateGov(govCcgpModel);
            } else {
                govCcgpDao.insertGov(govCcgpModel);
            }
            govCcgpModel.setIsEnclosure(null);
            //公告正文
            Elements textEle = document.select(".main .main_container .vF_detail_main");
            noticeText = textEle.text();
            if (noticeText != null && noticeOutline.length() != 0) {
                //存入公告正文表
                govTextModel.setGovId(govId);
                govTextModel.setText(noticeText);
                govTextDao.insertGovText(govTextModel);
            }
        }
    } catch (Exception e) {
        System.out.println(e.getMessage());
        String message = e.getMessage();
        if (message != null) {
            fetchException(message, id, pageSize, fetchUrl,is_homePage);
        }
    }
}

public void fetchException(String message, Integer id, Integer pageSize, String fetchUrl,Integer homePage) {
try {
Thread.sleep(60 * 1000);
if (message.equals(“HTTP error fetching URL”)) {

                    if(homePage == 0){
                        fetchHomePageUrl(fetchUrl);
                    }else {
                        fetchingPageUrl(id, pageSize, fetchUrl);
                    }
                }
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
}