关闭

用httpclient抓取全国火车票信息

标签: httpclient抓取火车票java
193人阅读 评论(0) 收藏 举报
分类:

一个通过httpclient抓取火车票信息的程序,需要修改下才能跑通,需要自己封装下httpclient,然后用get方式调用,还有fastJson,需要自己去解析下获得的数据,catchTrainInfo()是入口方法

import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;


import javax.annotation.Resource;


import org.slf4j.Logger;
import org.slf4j.LoggerFactory;


import java.util.Set;


public class CatchTrainInfo {

/**
     * 抓取火车票信息
     * @param attachmentId
     * @return
     */
    @SuppressWarnings("unchecked")
    public  String catchTrainInfo() {
    //车站信息
    HashSet<String> station = new HashSet<String>();
    //临时车站信息
    HashSet<String> stationTmp = new HashSet<String>();
    //车次信息
    Map<String,String> train = new HashMap<String,String>();

    String stationHtml = HttpClientUtil.get("http://www.59178.com/zhan/");

    String stationName = "";
    //截取车站信息
    String stations[] = stationHtml.split("htm'>");
    for (int i = 0; i < stations.length; i++) {
    if (i == 0 ) {
    continue;
    }
    stationName = getStation(stations[i]);
    if("".equals(stationName)) {
    continue;
    }
    station.add(stationName);
    }

    //循环,根据车站信息获取车次信息
    getTrainsByStation(station,train);

    //循环,根据车次获取车次详情,并保存入库
    getTrainDetailsByTrains(stationTmp,train);

    //继续执行3次循环,(本来应该stationTmp.size()==0的时候,防止无限循环)
    int loop = 3;
    while (loop > 0) {
            //stationTmp 和 station比较,stationTmp去掉已经遍历过的站点,station用stationTmp替代,继续循环
    HashSet<String> stationTmp1 = (HashSet<String>) stationTmp.clone();

            for (String stationTmpElement : stationTmp) {
             for (String stationElement : station) {
                  if(stationTmpElement.equals(stationElement)) {
                  stationTmp.remove(stationTmpElement);
                  }
                  }
            }
            station = stationTmp1;

        //循环,根据车站信息获取车次信息
        getTrainsByStation(stationTmp,train);

        //循环,根据车次获取车次详情,并写入数据库
        getTrainDetailsByTrains(stationTmp,train);

            loop--;
    }

return "success!";

    }


    /**
     * 根据车次获取车次详情,并写入数据库
     * @param stationTmp
     * @param train
     */
    private  void getTrainDetailsByTrains(HashSet<String> stationTmp, Map<String, String> train) {
    Iterator<Entry<String, String>> iterator = train.entrySet().iterator();
        while (iterator.hasNext()) {
        Entry<String, String> entry = iterator.next();
            String key =  entry.getKey();
            String value =  entry.getValue();
            if("unused".equals(value)) {
            getAndSaveTrainsDetails(key,stationTmp);
            entry.setValue("used");
            }
            try {
            //休眠一会,防止反爬虫
Thread.sleep(100);
} catch (InterruptedException e) {
e.printStackTrace();
}
       }
}


/**
     * 根据车站信息获取车次信息
     * @param station
     * @param train
     */
    private  void getTrainsByStation(HashSet<String> station, Map<String, String> train) {
    Iterator<String> iterator = station.iterator();
        while (iterator.hasNext()) {
            String key = iterator.next();
            getTrainsInfo(train,key);
            try {
            //休眠一会,防止反爬虫
Thread.sleep(100);
} catch (InterruptedException e) {
e.printStackTrace();
}
       }

}




/**
* 根据车站信息得到车次信息
* @param train
* @param stationName 
* @return
*/
private  void getTrainsInfo(Map<String, String> train, String stationName) {
try {
stationName = URLEncoder.encode(stationName,"utf-8");
} catch (UnsupportedEncodingException e1) {
log.error("转码出错了!stationName:" + stationName);
} 
String url = "http://train.qunar.com/qunar/stationInfo.jsp?q=" + stationName + "&format=json";
    //根据车站信息获取车次  ticketInfo
    String ticketInfo = HttpClientUtil.get(url);
        try{
            @SuppressWarnings("unchecked")
java.util.Map<String, Object> ticketInfos = (Map<String, Object>) JSONObject.parseObject(ticketInfo, java.util.Map.class).get("ticketInfo");
            Set<Entry<String, Object>> entries = ticketInfos.entrySet( );
            if (entries != null) {
           Iterator<Entry<String, Object>> iterator = entries.iterator( );
           while (iterator.hasNext( )) {
             Entry<String, Object> entry = iterator.next();
                String key = (String) entry.getKey( );
                String trains[] = key.split("/");
                for(int i = 0; i< trains.length; i++) {

train.put(trains[i], "unused");
                }
           }
            }
        } catch (Exception e) {
        log.info("根据车站信息得到车次信息报错:"+e.getMessage());
        }
}



/**
* 解析得到车站信息
* @param str
* @return
*/
private  String getStation(String str) {
if (str == null || str.length() <= 0) {
return "";
}
int pos = -1;
pos = str.indexOf("</a>", pos + 1);
if (pos == -1) {
return "";
}
return str.substring(0, pos);
}

/**

* 根据车次获取车次详情,并保存入库

* 

* @param key

* @param stationTmp

*/ 
public void getAndSaveTrainsDetails(String key, HashSet<String> stationTmp) {


String url = "http://train.qunar.com/qunar/checiInfo.jsp?q=" + key + "&date=20170107&format=json";
String trainScheduleBody = HttpClientUtil.getUtf8(url);
try {
List<Object> ticketInfos = (List<Object>) JSONObject.parseObject(trainScheduleBody, Map.class)
.get("trainScheduleBody");


for (int i = 0; i < ticketInfos.size(); i++) {
List<String> list = (List<String>) JSONObject.parseObject(ticketInfos.get(i).toString(), Map.class)
.get("content");
if (list == null || list.size() <= 0) {
continue;
}
stationTmp.add(list.get(1));

// 得到详情,更新入库
//TODO  
}


} catch (Exception e) {
log.info("根据车次获取车次详情报错:" + e.getMessage());
}
}

}
0
0

查看评论
* 以上用户言论只代表其个人观点,不代表CSDN网站的观点或立场
    个人资料
    • 访问:129539次
    • 积分:1582
    • 等级:
    • 排名:千里之外
    • 原创:22篇
    • 转载:143篇
    • 译文:0篇
    • 评论:10条
    最新评论