java爬取lol官网英雄名 与 故事

需要的maven坐标

  <!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>4.5.12</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.json/json -->
        <dependency>
            <groupId>org.json</groupId>
            <artifactId>json</artifactId>
            <version>20160810</version>
        </dependency>

        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>1.18.12</version>
            <scope>provided</scope>
        </dependency>

注意：不知道HttpClients可以先查看这篇文章
HttpClients

java程序

创建一个类来接收数据

package cn.guoke.pojo;

import lombok.Data;

@Data
public class Hero {
    private String heroId;
    private String Name;
    private String title;
    private String shortBio;

}

编写爬虫

package cn.guoke;


import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.json.JSONArray;
import org.json.JSONObject;
import 第一个爬虫程序.pojo.Hero;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

public class HeroUtils  {

    /**
     * @return 返回HeroId
     */
    public static List<Integer> getHeroId()  {
        CloseableHttpClient httpClient = HttpClients.createDefault();
        HttpGet httpGet = new HttpGet("https://game.gtimg.cn/images/lol/act/img/js/heroList/hero_list.js");
        ArrayList<Integer> integers = new ArrayList<Integer>();
        try {
            CloseableHttpResponse response = httpClient.execute(httpGet);
            if (response.getStatusLine().getStatusCode()==200){
                HttpEntity entity = response.getEntity();
                String s = EntityUtils.toString(entity, "utf-8");
                JSONObject jsonObject = new JSONObject(s);
                JSONArray hero = jsonObject.getJSONArray("hero");
                for (int i = 0; i < hero.length(); i++) {
                    int heroId = hero.getJSONObject(i).getInt("heroId");
                    integers.add(heroId);
                }

            }
        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            try {
                httpClient.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

        return integers;
    }


    /**
     *
     * @return 返回爬链接
     */
    public static List<String> getLolUrl(){
        //"https://game.gtimg.cn/images/lol/act/img/js/hero/1.js"
        ArrayList<String> strings = new ArrayList<String>();
        for (Integer integer : getHeroId()) {
            String s = integer.toString();
            String s1 = "https://game.gtimg.cn/images/lol/act/img/js/hero/"+s+".js";
            strings.add(s1);
//            System.out.println(s1);
        }

        return strings;
    }

    public static List<Hero> getAllHero(){
        CloseableHttpClient httpClient = HttpClients.createDefault();

        ArrayList<Hero> heroes = new ArrayList<Hero>();
        for (String s : getLolUrl()) {
            HttpGet httpGet = new HttpGet(s);
            try {
                CloseableHttpResponse httpResponse = httpClient.execute(httpGet);
                if (httpResponse.getStatusLine().getStatusCode()==200){
                    HttpEntity entity = httpResponse.getEntity();
                    String s1 = EntityUtils.toString(entity, "utf-8");
                    JSONObject jsonObject = new JSONObject(s1);
                    JSONObject hero = jsonObject.getJSONObject("hero");
                    String shortBio = hero.getString("shortBio");
                    String name = hero.getString("name");
                    String title = hero.getString("title");
                    String heroId = hero.getString("heroId");
                    Hero hero1 = new Hero();
                    hero1.setName(name);
                    hero1.setShortBio(shortBio);
                    hero1.setTitle(title);
                    hero1.setHeroId(heroId);
                    heroes.add(hero1);
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }


        return heroes;
    }


    public static void main(String[] args) {
        List<Hero> allHero = getAllHero();
        System.out.println(allHero);

    }
}