爬虫代码范例

JAVA爬虫（爬取东方财富网）

---

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.example</groupId>
    <artifactId>Spider</artifactId>
    <version>1.0-SNAPSHOT</version>
    <dependencies>
        <dependency>
            <groupId>net.sourceforge.htmlunit</groupId>
            <artifactId>htmlunit</artifactId>
            <version>2.49.1</version>
        </dependency>
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.76</version>
        </dependency>
        <dependency>
            <groupId>commons-io</groupId>
            <artifactId>commons-io</artifactId>
            <version>2.8.0</version>
        </dependency>
    </dependencies>
    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
    </properties>
</project>

代码源码

package spider;

import com.alibaba.fastjson.JSONObject;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.WebClient;
import org.apache.commons.io.FileUtils;

import java.io.File;
import java.io.IOException;

public class Spider_List {
    public static void main(String[] args) throws IOException {
        WebClient webClient = new WebClient(BrowserVersion.CHROME);
        String url = "http://26.push2.eastmoney.com/api/qt/clist/get?" +
                "pn=1" +
                "&pz=5000" +
                "&po=0" +
                "&np=1" +
                "&fltt=2" +
                "&invt=2" +
                "&fid=f12" +
                "&fs=m:0+t:6,m:0+t:80,m:1+t:2,m:1+t:23" +
                "&fields=f12,f13";
        String json = webClient.getPage(url).getWebResponse().getContentAsString();
        Object[] objects = JSONObject.parseObject(json).getJSONObject("data").getJSONArray("diff").toArray();
        for (Object object : objects) {
            String F12 = JSONObject.parseObject(object.toString()).getString("f12");
            String F13 = JSONObject.parseObject(object.toString()).getString("f13");
            String url1 = "http://12.push2his.eastmoney.com/api/qt/stock/kline/get?secid=" + F13 + "." + F12 + "&fields1=f1%2Cf2%2Cf3%2Cf4%2Cf5%2Cf6&fields2=f51%2Cf52%2Cf53%2Cf54%2Cf55%2Cf56%2Cf57%2Cf58%2Cf59%2Cf60%2Cf61&klt=60&fqt=0&end=20500101&lmt=120";
            String json2 = webClient.getPage(url1).getWebResponse().getContentAsString();
            System.out.println(json2);
            FileUtils.writeStringToFile(new File("C:\\Users\\LN\\Desktop\\data_60.csv"), json2 + "\n", "UTF-8", true);
        }
        System.out.println("总共" + objects.length + "条数据");
    }
}