java 吉时开奖网数据爬取中奖的数据

今天有个人要爬取这个网站的数据，地址为：

https://ig185.com/mkjw/html/PCEGG_180/index.html

然后我分析了这个网站的代码，发现这个网站是用structs来写的，然后分析网络连接发现，可以直接调用url：

https://ig185.com/kaijiangweb/getHistoryList.do?date=&lotCode=PCEGG_180

就可以获取到该网页的数据了，由于数据已经是json的数据，我们用fastjson封装编程类数据，我创建的是maven项目，依赖为：

   <!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
    <dependency>
        <groupId>org.apache.httpcomponents</groupId>
        <artifactId>httpclient</artifactId>
        <version>4.5.3</version>
    </dependency>
    <dependency>
        <groupId>com.alibaba</groupId>
        <artifactId>fastjson</artifactId>
        <version>1.2.13</version>
    </dependency>

创建javabean，首先是result.java

package bean;

import java.util.List;

public class Result {
   private List<Lottery> result;
   private int errorCode;
   private String message;

    public List<Lottery> getResult() {
        return result;
    }

    public void setResult(List<Lottery> result) {
        this.result = result;
    }

    public int getErrorCode() {
        return errorCode;
    }

    public void setErrorCode(int errorCode) {
        this.errorCode = errorCode;
    }

    public String getMessage() {
        return message;
    }

    public void setMessage(String message) {
        this.message = message;
    }
}

lottery.java类

package bean;

import java.lang.reflect.Array;
import java.util.Arrays;

public class Lottery {
    private int sumSingleDouble;
    private int sumBigSmall;
    private String preDrawIssue;
    private String preDrawTime;
    private int sumNum;
    private int groupCode;
    private int[] preDrawCode;

    public int getSumSingleDouble() {
        return sumSingleDouble;
    }

    public void setSumSingleDouble(int sumSingleDouble) {
        this.sumSingleDouble = sumSingleDouble;
    }

    public int getSumBigSmall() {
        return sumBigSmall;
    }

    public void setSumBigSmall(int sumBigSmall) {
        this.sumBigSmall = sumBigSmall;
    }

    public String getPreDrawIssue() {
        return preDrawIssue;
    }

    public void setPreDrawIssue(String preDrawIssue) {
        this.preDrawIssue = preDrawIssue;
    }

    public String getPreDrawTime() {
        return preDrawTime;
    }

    public void setPreDrawTime(String preDrawTime) {
        this.preDrawTime = preDrawTime;
    }

    public int getSumNum() {
        return sumNum;
    }

    public void setSumNum(int sumNum) {
        this.sumNum = sumNum;
    }

    public int getGroupCode() {
        return groupCode;
    }

    public void setGroupCode(int groupCode) {
        this.groupCode = groupCode;
    }

    public int[] getPreDrawCode() {
        return preDrawCode;
    }

    public void setPreDrawCode(int[] preDrawCode) {
        this.preDrawCode = preDrawCode;
    }

    @Override
    public String toString() {
        return "Lottery{" +
                "sumSingleDouble=" + sumSingleDouble +
                ", sumBigSmall=" + sumBigSmall +
                ", preDrawIssue='" + preDrawIssue + '\'' +
                ", preDrawTime='" + preDrawTime + '\'' +
                ", sumNum=" + sumNum +
                ", groupCode=" + groupCode +
                ", preDrawCode=" + Arrays.toString(preDrawCode) +
                '}';
    }
}

然后是LotteryPaser.java类

import bean.Lottery;
import bean.Result;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.TypeReference;
import org.apache.http.HttpEntity;
import org.apache.http.ParseException;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;
import java.util.List;

public class LotteryPaser {
    public static void main(String args[]){
        String url="https://ig185.com/kaijiangweb/getHistoryList.do?date=&lotCode=PCEGG_180";
        String html=pickData(url);
        System.out.println(html);
        Result result = JSON.parseObject(html, new TypeReference<Result>() {});
        System.out.println(result.getMessage());
        System.out.println(result.getResult());
        List<Lottery> lotteries=result.getResult();
        for(Lottery lottery:lotteries){
            System.out.println(lottery.toString());
            System.out.println(lottery.getPreDrawTime());
        }

    }

    /*
     * 爬取网页信息
     */
    private static String pickData(String url) {
        CloseableHttpClient httpclient = HttpClients.createDefault();
        try {
            HttpGet httpget = new HttpGet(url);
            CloseableHttpResponse response = httpclient.execute(httpget);
            try {
                // 获取响应实体
                HttpEntity entity = response.getEntity();
                // 打印响应状态
                if (entity != null) {
                    return EntityUtils.toString(entity);
                }
            } finally {
                response.close();
            }
        } catch (ClientProtocolException e) {
            e.printStackTrace();
        } catch (ParseException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            // 关闭连接,释放资源
            try {
                httpclient.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return null;
    }

}

参考文献

[1]. 使用java jsoup抓取页面中的数据. https://blog.csdn.net/qq_18115729/article/details/79239970

[2].使用java开源工具httpClient及jsoup抓取解析网页数据.https://www.cnblogs.com/lkxsnow/p/5380164.html

[3].FastJson对于JSON格式字符串、JSON对象及JavaBean之间的相互转换. https://www.cnblogs.com/cdf-opensource-007/p/7106018.html