Java数据爬取——爬取携程酒店数据（二）

在上篇文章Java数据爬取——爬取携程酒店数据（一）爬取所有地区后，继续根据地区数据爬取酒店数据

1.首先思考怎样根据地域获取地域酒店信息，那么我们看一下携程上是怎样获得的。
还是打开http://hotels.ctrip.com/domestic-city-hotel.html 这个地址，随便点击一个地区进去（这里我选取澳门作为示例），点击第二页数据，很高兴发现

http://hotels.ctrip.com/Domestic/Tool/AjaxHotelList.aspx 传递参数给这个Url可以得到酒店数据

2.模拟请求，获取数据

从这里可以发现，是POST请求，请求的body中有很多参数，

所以先封装一下参数，经过测试，有些参数可以不用传递，这里写了一个方法，模拟请求

public String getHotelListString(HotelCity city, String page){
        HashMap<String, String> params = new HashMap<String, String>();
        params.put("__VIEWSTATEGENERATOR", "DB1FBB6D");
        params.put("cityName", city.getCityName());
//      params.put("StartTime", "2016-11-24");
//      params.put("DepTime", "2016-11-25");
        params.put("txtkeyword", "");
        params.put("Resource", "");
        params.put("Room", "");
        params.put("Paymentterm", "");
        params.put("BRev", "");
        params.put("Minstate", "");
        params.put("PromoteType", "");
        params.put("PromoteDate", "");
        params.put("operationtype", "NEWHOTELORDER");
        params.put("PromoteStartDate", "");
        params.put("PromoteEndDate", "");
        params.put("OrderID", "");
        params.put("RoomNum", "");
        params.put("IsOnlyAirHotel", "F");
        params.put("cityId", city.getCityId());
        params.put("cityPY", city.getPinyin());
//      params.put("cityCode", "1853");
//      params.put("cityLat", "22.1946");
//      params.put("cityLng", "113.549");
        params.put("positionArea", "");
        params.put("positionId", "");
        params.put("keyword", "");
        params.put("hotelId", "");
        params.put("htlPageView", "0");
        params.put("hotelType", "F");
        params.put("hasPKGHotel", "F");
        params.put("requestTravelMoney", "F");
        params.put("isusergiftcard", "F");
        params.put("useFG", "F");
        params.put("HotelEquipment", "");
        params.put("priceRange", "-2");
        params.put("hotelBrandId", "");
        params.put("promotion", "F");
        params.put("prepay", "F");
        params.put("IsCanReserve", "F");
        params.put("OrderBy", "99");
        params.put("OrderType", "");
        params.put("k1", "");
        params.put("k2", "");
        params.put("CorpPayType", "");
        params.put("viewType", "");
//      params.put("checkIn", "2016-11-24");
//      params.put("checkOut", "2016-11-25");
        params.put("DealSale", "");
        params.put("ulogin", "");
        params.put("hidTestLat", "0%7C0");
//      params.put("AllHotelIds", "436450%2C371379%2C396332%2C419374%2C345805%2C436553%2C425997%2C436486%2C436478%2C344977%2C5605870%2C344983%2C371396%2C344979%2C2572033%2C699384%2C425795%2C419823%2C2010726%2C5772619%2C1181591%2C2005951%2C345811%2C371381%2C371377");// TODO
        params.put("psid", "");
        params.put("HideIsNoneLogin", "T");
        params.put("isfromlist", "T");
        params.put("ubt_price_key", "htl_search_result_promotion");
        params.put("showwindow", "");
        params.put("defaultcoupon", "");
        params.put("isHuaZhu", "False");
        params.put("hotelPriceLow", "");
        params.put("htlFrom", "hotellist");
        params.put("unBookHotelTraceCode", "");
        params.put("showTipFlg", "");
//      params.put("hotelIds", "436450_1_1,371379_2_1,396332_3_1,419374_4_1,345805_5_1,436553_6_1,425997_7_1,436486_8_1,436478_9_1,344977_10_1,5605870_11_1,344983_12_1,371396_13_1,344979_14_1,2572033_15_1,699384_16_1,425795_17_1,419823_18_1,2010726_19_1,5772619_20_1,1181591_21_1,2005951_22_1,345811_23_1,371381_24_1,371377_25_1");// TODO
        params.put("markType", "1");
        params.put("zone", "");
        params.put("location", "");
        params.put("type", "");
        params.put("brand", "");
        params.put("group", "");
        params.put("feature", "");
        params.put("equip", "");
        params.put("star", "");
        params.put("sl", "");
        params.put("s", "");
        params.put("l", "");
        params.put("price", "");
        params.put("a", "0");
        params.put("keywordLat", "");
        params.put("keywordLon", "");
        params.put("contrast", "0");
        params.put("page", page);
        params.put("contyped", "0");
        params.put("productcode", "");
        String result = HttpUtil.getInstance().httpPost(hotelUrl, params);
        // 数据中有转义符直接转JSON报错，所以这里重新拼接所需要的JSON数据
        String tempHotel = result.substring(result.indexOf("hotelPositionJSON")-1, result.length());
        // 确保截取到indexOf("biRecord"), 减2是因为需要]符号
        String hotelArray = tempHotel.substring(0, tempHotel.indexOf("biRecord") - 2);
        String tempTotalCount = result.substring(result.indexOf("hotelAmount")-1, result.length());
        String totalCount = tempTotalCount.substring(0, tempTotalCount.indexOf(","));
        StringBuffer sb = new StringBuffer();
        sb.append("{");
        sb.append(totalCount);
        sb.append(",");
        sb.append(hotelArray);
        sb.append("}");
        return sb.toString().replace("\\", "");
    }

说明：这个方法对返回结果进行了操作，直接把hotelPositionJSON和hotelAmount的内容提取了出来（因为直接转换时发现数据中含有转义符导致转为JSON失败），操作后的数据格式为：

{
    "hotelAmount": 11265,
    "hotelPositionJSON": [
        {
            "id": "6297824",
            "name": "北京浣川招待所",
            "lat": "39.918086",
            "lon": "116.427508",
            "url": "/hotel/6297824.html?isFull=F#ctm_ref=hod_sr_map_dl_txt_1",
            "img": "http://pic.c-ctrip.com/hotels110127/hotel_example.jpg",
            "address": "东城区东城区外交部街46号-1。 （ 北京站、建国门地区）",
            "score": "0.0",
            "dpscore": "0",
            "dpcount": "0",
            "star": "hotel_diamond01",
            "stardesc": "携程用户评定为1钻",
            "shortName": "",
            "isSingleRec": "false"
        },
        {
            "id": "6298279",
            "name": "北京怀柔喇叭沟门海燕农家院",
            "lat": "40.956471",
            "lon": "116.513108",
            "url": "/hotel/6298279.html?isFull=F#ctm_ref=hod_sr_map_dl_txt_2",
            "img": "http://pic.c-ctrip.com/hotels110127/hotel_example.jpg",
            "address": "怀柔区黄甸子村。 （ 怀柔风景区）",
            "score": "0.0",
            "dpscore": "0",
            "dpcount": "0",
            "star": "hotel_diamond01",
            "stardesc": "携程用户评定为1钻",
            "shortName": "",
            "isSingleRec": "false"
        }
    ]
}

3.循环遍历城市数据，查询每个城市的酒店即可

long startTime = System.currentTimeMillis();
HotelCitySpider citySpider = new HotelCitySpider();
HotelSpider spider = new HotelSpider();
List<HotelCity> cities = citySpider.getHotelCities();
long getCityTime = System.currentTimeMillis();
System.out.println("获取城市所用时间(ms):" + (getCityTime - startTime));
spider.createTable();
for (HotelCity hotelCity : cities) {
    spider.saveHotels(hotelCity, spider.getHotelList(hotelCity));
}
long saveHotelTime = System.currentTimeMillis();
System.out.println("获取酒店并存储所用时间(ms):" + (saveHotelTime - startTime));

这里我仍然将数据存储到数据库中

4.批量插入数据

/**
     * 保存每个城市的酒店列表
     * @param city
     * @param hotels
     */
    public void saveHotels(HotelCity city, List<Hotel> hotels) {
        for (Hotel hotel : hotels) {
            StringBuffer insert_sql = new StringBuffer();
            insert_sql.append("insert into ctrip_hotel "
                    + "(hotel_id, city_id, city_name, name, lat, lon, url, img, address, score, dpscore, dpcount, star, stardesc, shortName, isSingleRec) values (");
            insert_sql.append("'" + hotel.getId() + "'");
            insert_sql.append(", " + city.getCityId());
            insert_sql.append(", '" + city.getCityName() + "'");
            insert_sql.append(", '" + hotel.getName() + "'");
            insert_sql.append(", " + hotel.getLat());
            insert_sql.append(", " + hotel.getLon());
            insert_sql.append(", '" + hotel.getUrl() + "'");
            insert_sql.append(", '" + hotel.getImg() + "'");
            insert_sql.append(", '" + hotel.getAddress() + "'");
            insert_sql.append(", " + hotel.getScore());
            insert_sql.append(", " + hotel.getDpscore());
            insert_sql.append(", " + hotel.getDpcount());
            insert_sql.append(", '" + hotel.getStar() + "'");
            insert_sql.append(", '" + hotel.getStardesc() + "'");
            insert_sql.append(", '" + hotel.getShortName() + "'");
            insert_sql.append(", " + hotel.getIsSingleRec() + ")");
            try {
                preparedStatement = conn.prepareStatement(insert_sql.toString());
                preparedStatement.execute();
            } catch (Exception e) {
                e.getMessage();
                continue;
            }
        }
    }

原本想批量插入，但是数据有重复的hotel_id，preparedStatement.executeBatch()会报错，所以还是一条条插入。
github源码地址 https://github.com/jianiuqi/CTripSpider