JAVA爬虫---LOL各英雄图片（含皮肤）下载

pom依赖

 		<dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>4.5.2</version>
        </dependency>
          <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.69</version>
        </dependency>
         <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

实现代码

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.apache.http.impl.client.HttpClientBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.task.AsyncTaskExecutor;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.util.StringUtils;
import org.springframework.web.client.RestTemplate;

import javax.imageio.stream.FileImageOutputStream;
import javax.imageio.stream.ImageOutputStream;
import java.io.File;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadPoolExecutor;

/**
 * @author chunyang.leng
 * @date 2021/1/31 7:57 下午
 */
public class LOLImagesDownload {
    private static String listURL = "http://game.gtimg.cn/images/lol/act/img/js/heroList/hero_list.js";
    private static String imageInfoURL = "http://game.gtimg.cn/images/lol/act/img/js/hero/{0}.js";
    private static List<Future<String>> list = Collections.synchronizedList(new ArrayList<>());

    private static final Logger logger = LoggerFactory.getLogger(LOLImagesDownload.class);

    /**
     * 需要调整的地方，配置你需要保存的路径，以 / 结束
     */
    private static String saveDir = "/Users/lcy/Pictures/LOL壁纸/";

    public static void main(String[] args) {
        RestTemplate restTemplate = getRestTemplate();
        AsyncTaskExecutor threadPool = getThreadPool();

        ResponseEntity<String> responseEntity = restTemplate.getForEntity(listURL, String.class);
        String stringBody = responseEntity.getBody();
        JSONObject body = JSONObject.parseObject(stringBody);
        JSONArray heroList = body.getJSONArray("hero");
        for (int i = 0; i < heroList.size(); i++) {
            JSONObject hero = heroList.getJSONObject(i);
            String heroId = hero.getString("heroId");
            String name = hero.getString("name");

            File dir = new File(saveDir + name);
            if (!dir.exists()) {
                dir.mkdirs();
            }
            String imageUrl = MessageFormat.format(imageInfoURL, heroId);
            ResponseEntity<String> forEntity = restTemplate.getForEntity(imageUrl, String.class);
            String imageInfoBody = forEntity.getBody();
            JSONObject imageInfo = JSONObject.parseObject(imageInfoBody);
            JSONArray skins = imageInfo.getJSONArray("skins");
            for (int i1 = 0; i1 < skins.size(); i1++) {
                int finalI = i1;
                Future<String> future = threadPool.submit(() -> {
                    JSONObject jsonObject = skins.getJSONObject(finalI);
                    String mainImgUrl = jsonObject.getString("mainImg");
                    if (StringUtils.isEmpty(mainImgUrl)) {
                        return null;
                    }
                    String fileName = jsonObject.getString("name").replaceAll("\\s", "-");

                    File image = new File(dir + "/" + fileName + ".jpg");
                    if (!image.exists()) {
                        image.createNewFile();
                    } else {
                        image.delete();
                        image.createNewFile();
                    }
                    HttpHeaders headers = new HttpHeaders();
                    ResponseEntity<byte[]> response = restTemplate.exchange(
                            mainImgUrl,
                            HttpMethod.GET,
                            new HttpEntity<byte[]>(headers),
                            byte[].class);

                    byte[] result = response.getBody();
                    ImageOutputStream imageOutputStream = new FileImageOutputStream(image);
                    imageOutputStream.write(result);
                    imageOutputStream.flush();
                    imageOutputStream.close();
                    return fileName;
                });
                list.add(future);
            }
        }
        list.forEach(o -> {
            if (o != null) {
                try {
                    final String s = o.get();
                    if (o != null) {
                        logger.info(s + ":下载成功");
                    }
                } catch (Exception e) {
                    logger.error("下载出错", e);
                }
            }
        });


    }

    public static AsyncTaskExecutor getThreadPool() {
        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
        //获取到服务器的cpu内核
        int i = Runtime.getRuntime().availableProcessors();
        System.out.println("cpu core :" + i);
        //核心池大小
        executor.setCorePoolSize(i * 2);
        //最大线程数
        executor.setMaxPoolSize(i * 4);
        //线程空闲时间
        executor.setKeepAliveSeconds(100000);
        //线程前缀名称
        executor.setThreadNamePrefix("pool-");
        //配置拒绝策略
        executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
        executor.afterPropertiesSet();
        return executor;
    }

    private static RestTemplate getRestTemplate() {
        return new RestTemplate(new HttpComponentsClientHttpRequestFactory(
                HttpClientBuilder.create().build()));
    }
}

效果截图