Android抓取百度图片

最新推荐文章于 2021-02-01 19:30:16 发布

Chandler Coder

最新推荐文章于 2021-02-01 19:30:16 发布

阅读量854

点赞数

分类专栏： Android 文章标签：爬取百度图片 Android爬虫 MVP

本文链接：https://blog.csdn.net/zengcq2016/article/details/99683538

版权

Android 专栏收录该内容

2 篇文章

订阅专栏

本项目介绍如何抓取百度图片数据、输入搜索关键字即可实现百度图片搜索并显示在app页面上。

github源码地址：https://github.com/ChandlerZeng/MyStock.git

安卓App,欢迎扫码体验：

抓取百度图片app展示：app点击我的-网络图片，即可进入该页面

本项目采用MVP架构，实现业务逻辑与页面显示、数据处理进行分离，达到项目松耦合高内聚的设计要求。

网络请求采用Volley框架，具体封装如下：

Presenter接口类，主要定义了网络请求的各个生命周期方法

/**
 * @author zcq
 */

public interface Presenter {
    void onCreate();

    void onStart();

    void onStop();

    void pause();

    void attachView(BaseView view);

    void attachIncomingIntent(Intent intent);
}

//Presenter具体实现类
public class CrawlerPresenter implements Presenter{
    private static final String URL = "https://image.baidu.com/search/acjson";//百度图片搜索URL
   
    private Context context;
    HttpManager manager;
    HttpResponseView<SimpleCrawerResult> httpResponseView;

    public CrawlerPresenter(Context context) {
        this.context = context;
    }

    @Override
    public void onCreate() {
        manager = HttpManager.getInstance();
    }

    @Override
    public void onStart() {

    }

    @Override
    public void onStop() {

    }

    @Override
    public void pause() {

    }

    @Override
    public void attachView(BaseView view) {
        httpResponseView = (HttpResponseView<SimpleCrawerResult>) view;
    }

    @Override
    public void attachIncomingIntent(Intent intent) {

    }

    public void getBaiduImages(CrawerSimpleParams queryParams){
        String url = addGetParams(queryParams);
// Volley网络请求
        StringRequest stringRequest = new StringRequest(Request.Method.POST, url, new Response.Listener<String>() {

            @Override
            public void onResponse(String response) {
//请求结果为json字符串，对其进行解析
                Gson gson = new Gson();
                SimpleCrawerResult crawerResult = gson.fromJson(response,SimpleCrawerResult.class);
                Log.e("Gson","first url:"+crawerResult.getData().get(0).getMiddleURL());
                httpResponseView.onSuccess(crawerResult);
            }
        }, new Response.ErrorListener() {
            @Override
            public void onErrorResponse(VolleyError error) {
                httpResponseView.onError(error.getMessage());
            }
        });
        Log.i("Crawler","originUrl:"+stringRequest.getOriginUrl());
        Log.i("Crawler","Url:"+stringRequest.getUrl());
        manager.startStringRequest(stringRequest);
    }

    private String addGetParams(CrawerSimpleParams queryParams){
//添加请求参数，word，queryWord为搜索关键字，pn为总数，rn为每页数量，其它参数都使用默认的值即可
        Map<String, String> params = new HashMap<>();
        params.put("tn", "resultjson_com");
        params.put("ipn", "rj");
        params.put("ct", "201326592");
        params.put("fp", "result");
        params.put("queryWord", queryParams.getQueryWord());//搜索关键字
        params.put("cl", "2");
        params.put("lm", "-1");
        params.put("ie", "utf-8");
        params.put("oe", "utf-8");
        params.put("word", queryParams.getWord());//搜索关键字
        params.put("pn", queryParams.getPn());//总数
        params.put("rn", queryParams.getRn());//每页数量
        params.put("gsm", "5a");
        StringBuilder sb = new StringBuilder();
        sb.append(URL).append("?");
        for (Map.Entry<String, String> entry : params.entrySet()) {
            // 如果请求参数中有中文，需要进行URLEncoder编码 gbk/utf8
            try {
                sb.append(entry.getKey()).append("=").append(URLEncoder.encode(entry.getValue(), "utf-8"));
            } catch (UnsupportedEncodingException e) {
                e.printStackTrace();
            }
            sb.append("&");
        }
        sb.deleteCharAt(sb.length() - 1);
        return sb.toString();
    }
}

请求参数实体类Model

public class CrawerSimpleParams {

    private String queryWord;
    private String word;
    private String pn;
    private String rn;

    public String getQueryWord() {
        return queryWord;
    }

    public void setQueryWord(String queryWord) {
        this.queryWord = queryWord;
    }

    public String getWord() {
        return word;
    }

    public void setWord(String word) {
        this.word = word;
    }

    public String getPn() {
        return pn;
    }

    public void setPn(String pn) {
        this.pn = pn;
    }

    public String getRn() {
        return rn;
    }

    public void setRn(String rn) {
        this.rn = rn;
    }
}

请求结果实体类

public class SimpleCrawerResult {
    /**
     * queryEnc : %E5%A4%A7%E5%A5%B6%E7%BE%8E%E5%A5%B3
     * queryExt : 大奶美女
     * listNum : 595
     * displayNum : 595
     * gsm : 3
     * bdFmtDispNum : 595
     * bdSearchTime :
     * isNeedAsyncRequest : 0
     * bdIsClustered : 1
     * data : [{"adType":"0","hasAspData":"0","thumbURL":"https://ss0.bdstatic.com/70cFvHSh_Q1YnxGkpoWK1HF6hhy/it/u=2286768330,1999210542&fm=26&gp=0.jpg","middleURL":"https://ss0.bdstatic.com/70cFvHSh_Q1YnxGkpoWK1HF6hhy/it/u=2286768330,1999210542&fm=26&gp=0.jpg","largeTnImageUrl":"","hasLarge":0,"hoverURL":"https://ss0.bdstatic.com/70cFvHSh_Q1YnxGkpoWK1HF6hhy/it/u=2286768330,1999210542&fm=26&gp=0.jpg","pageNum":2,"objURL":"ippr_z2C$qAzdH3FAzdH3Ft42_z&e3Bp7rtwgz3_z&e3Bv54AzdH3F7rs5w1fAzdH3Fwsst42AzdH3F8ma88bAzdH3Fl-8ma88Pldc8l_z&e3B3r2","fromURL":"ippr_z2C$qAzdH3FAzdH3Fooo_z&e3Bp7rtwgz3_z&e3Bv54AzdH3F4jtgeAzdH3Fda8ma88bAzdH3Fnldnb_b_z&e3Bip4s","fromURLHost":"www.tupianzj.com","currentIndex":"","width":945,"height":1417,"type":"jpg","is_gif":0,"strategyAssessment":"263586_0_0_0","filesize":"","bdSrcType":"0","di":"5060","pi":"0","is":"0,0","imgCollectionWord":"","replaceUrl":[{"ObjURL":"http://img0.imgtn.bdimg.com/it/u=2286768330,1999210542&fm=214&gp=0.jpg","ObjUrl":"http://img0.imgtn.bdimg.com/it/u=2286768330,1999210542&fm=214&gp=0.jpg","FromURL":"http://www.tupianzj.com/meinv/20160118/39238_8.html","FromUrl":"http://www.tupianzj.com/meinv/20160118/39238_8.html"}],"hasThumbData":"0","bdSetImgNum":0,"partnerId":0,"spn":0,"bdImgnewsDate":"2016-01-18 00:00","fromPageTitle":"日本性感泳装大奶美女<\/strong>泳池写真(8)","fromPageTitleEnc":"日本性感泳装大奶美女泳池写真(8)","bdSourceName":"","bdFromPageTitlePrefix":"","isAspDianjing":0,"token":"","imgType":"","cs":"2286768330,1999210542","os":"911903161,1906406124","simid":"0,0","personalized":"0","simid_info":null,"face_info":null,"xiangshi_info":null,"adPicId":"0","source_type":""},{}]
     */

    private java.util.List<SimpleDataBean> data;

    public List<SimpleDataBean> getData() {
        return data;
    }

    public void setData(List<SimpleDataBean> data) {
        this.data = data;
    }
}
public class SimpleDataBean {

        private String thumbURL;
        private String middleURL;//图片URL
        private String objURL;
        private String fromURL;
        private String fromURLHost;
        private String currentIndex;
        private int width;
        private int height;
        private String type;
        private int is_gif;
        private String filesize;
        private String fromPageTitleEnc;

    public String getThumbURL() {
        return thumbURL;
    }

    public void setThumbURL(String thumbURL) {
        this.thumbURL = thumbURL;
    }

    public String getMiddleURL() {
        return middleURL;
    }

    public void setMiddleURL(String middleURL) {
        this.middleURL = middleURL;
    }

    public String getObjURL() {
        return objURL;
    }

    public void setObjURL(String objURL) {
        this.objURL = objURL;
    }

    public String getFromURL() {
        return fromURL;
    }

    public void setFromURL(String fromURL) {
        this.fromURL = fromURL;
    }

    public String getFromURLHost() {
        return fromURLHost;
    }

    public void setFromURLHost(String fromURLHost) {
        this.fromURLHost = fromURLHost;
    }

    public String getCurrentIndex() {
        return currentIndex;
    }

    public void setCurrentIndex(String currentIndex) {
        this.currentIndex = currentIndex;
    }

    public int getWidth() {
        return width;
    }

    public void setWidth(int width) {
        this.width = width;
    }

    public int getHeight() {
        return height;
    }

    public void setHeight(int height) {
        this.height = height;
    }

    public String getType() {
        return type;
    }

    public void setType(String type) {
        this.type = type;
    }

    public int getIs_gif() {
        return is_gif;
    }

    public void setIs_gif(int is_gif) {
        this.is_gif = is_gif;
    }

    public String getFilesize() {
        return filesize;
    }

    public void setFilesize(String filesize) {
        this.filesize = filesize;
    }

    public String getFromPageTitleEnc() {
        return fromPageTitleEnc;
    }

    public void setFromPageTitleEnc(String fromPageTitleEnc) {
        this.fromPageTitleEnc = fromPageTitleEnc;
    }
}

请求返回与Ui交互的HttpResponseView是一个接口类，具体实现在Activity中


public interface BaseView {
}

public interface HttpResponseView<T> extends BaseView{
    void onSuccess(T responseBody);
    void onError(String result);
}

Activity中具体使用如下：

private CrawerSimpleParams params;
    private CrawlerPresenter crawlerPresenter;
    private HttpResponseView<SimpleCrawerResult> httpResponseView = new HttpResponseView<SimpleCrawerResult>() {
        @Override
        public void onSuccess(SimpleCrawerResult responseBody) {
            if (responseBody != null) {
                hideKeybord();
                if (curPage == 1) {
                    photoBeanLists.clear();
                }
                curPage++;
                List<SimpleDataBean> dataBeans = responseBody.getData();
                if (dataBeans != null && dataBeans.size() > 0) {
                    for (SimpleDataBean bean : dataBeans) {
                        if (!TextUtils.isEmpty(bean.getMiddleURL())) {
                            PhotoBean photoBean = new PhotoBean();
                            photoBean.setUrl(bean.getMiddleURL());
                            photoBeanLists.add(photoBean);
                        }
                    }
                }
                photoAdapter.setData(photoBeanLists);
                if (photoBeanLists.size() > 100) {
                    mRecyclerView.setNoMore(true);
                } else {
                    mRecyclerView.setNoMore(false);
                }
                mRecyclerView.refreshComplete(20);
            }
            FileUtil.save("my_stock_images", GsonUtil.objectToJson(responseBody));
        }

        @Override
        public void onError(String result) {
            Log.i("Crawler", "error:" + result);
            mRecyclerView.refreshComplete(20);
        }
    };


    private void getBaiduPics(String queryWord, int offset) {
        params = new CrawerSimpleParams();
        params.setQueryWord(queryWord);
        params.setWord(queryWord);
        params.setRn("30");
        int pn = 30 * offset;
        params.setPn(pn + "");
        crawlerPresenter.getBaiduImages(params);
    }