JAVA爬取b站用户数据

用到的jar包
链接：https://pan.baidu.com/s/1UYi1fcrNc8xNoTdXsipsPw
提取码：csdn

使用的数据库为MySQL

工具类通过mid获得User包装类对象，以及将User类写入数据库

//工具类通过mid获得User包装类对象，以及将User类写入数据库
public class MidUtil {
    //通过mid获取User
    public static User getUser(int mid){
        User user = null;
        String following =null;
        String follower=null;
        String name =null;
        JsonObject userInformation = loadJson("https://api.bilibili.com/x/relation/stat?vmid="+mid);
        if("\"0\"".equals(userInformation.get("message").toString())){
            JsonObject Json_follow = StringToJsonObject(userInformation.get("data").toString());
            //获取关注数
            following = Json_follow.get("following").toString();
            //获取粉丝数
            follower = Json_follow.get("follower").toString();
        }
        JsonObject userName = loadJson("https://api.bilibili.com/x/space/acc/info?mid="+mid);
        if("\"0\"".equals(userName.get("message").toString())){
            JsonObject Json_name = StringToJsonObject(userName.get("data").toString());
            //包装User类
            name = Json_name.get("name").toString();
        }
        user = new User(mid,name,Integer.parseInt(following),Integer.parseInt(follower));
        return user;
    }
    //通过url获取JsonObject
    public static JsonObject loadJson (String url) {
        StringBuilder json = new StringBuilder();
        Gson g = new Gson();
        try {
            URL urlObject = new URL(url);
            URLConnection uc = urlObject.openConnection();
            BufferedReader in = new BufferedReader(new InputStreamReader(uc.getInputStream(),"utf-8"));
            String inputLine = null;
            while ( (inputLine = in.readLine()) != null) {
                json.append(inputLine);
            }
            in.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
        return g.fromJson(json.toString(), JsonObject.class);
    }
    //将String转为JsonObject
    public static JsonObject StringToJsonObject(String text){
        Gson g = new Gson();
        JsonObject jsonObject = g.fromJson(text, JsonObject.class);
        return  jsonObject;
    }
    //将User信息添加到数据库
    public static void addToDatabase(User[] user,String DatabaseName){
        JdbcTemplate template = new JdbcTemplate(JDBCUtils.getDataSource());
        String sql = "insert into "+DatabaseName+" values ";
        sql=sql+"("+user[0].getMid()+","+user[0].getName()+","+user[0].getFollowing()+","+user[0].getFollower()+")";
        for(int i=1;i<100;i++){
            sql= sql+",("+user[i].getMid()+","+user[i].getName()+","+user[i].getFollowing()+","+user[i].getFollower()+")";
        }
        template.update(sql);
    }
}

主程序，通过调用工具类达到目的

//主程序，通过调用工具类达到目的
ublic class bilibili {
    public static void main(String[] args) {
        int mid=1;
        User[] user=new User[100];
        //粗略估算b站用户约为703200000
        while(mid<703200000){
        	//每次插入一百条数据
            for(int i = 0 ;i<100;i++){
                user[i] = MidUtil.getUser(mid);
                //数据输出以便监控
                System.out.println(user[i]);
                mid++;
            }
            MidUtil.addToDatabase(user,"user");
        }
    }
}

数据库连接工具类，采用Druid数据库连接池，简化数据库连接，提高效率

//数据库连接工具类，采用Druid数据库连接池，简化数据库连接，提高效率
public class JDBCUtils {
    private static DataSource ds;

    static{
        Properties pr = new Properties();
        try {
            //加载配置文件
            pr.load(JDBCUtils.class.getClassLoader().getResourceAsStream("druid.properties"));
            //获取DataSource
            ds = DruidDataSourceFactory.createDataSource(pr);
        } catch (IOException e) {
            e.printStackTrace();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    //获取连接池(DataSource)
    public static DataSource getDataSource(){
        return ds;
    }
    //获取连接
    public static Connection getConnection() throws SQLException {
            return ds.getConnection();
    }
    //关闭资源
    public static void close(Statement state, Connection conn){
        if(state!=null){
            try {
                state.close();
            } catch (SQLException throwables) {
                throwables.printStackTrace();
            }
        }
        if(conn!=null){
            try {
                conn.close();
            } catch (SQLException throwables) {
                throwables.printStackTrace();
            }

        }
    }
    public static void close(ResultSet rs, Statement state, Connection conn){
        if(rs!=null){
            try {
                rs.close();
            } catch (SQLException throwables) {
                throwables.printStackTrace();
            }
        }
        close(state,conn);
    }
}

User包装类

public class User {
    private int mid;
    private String name;
    private int following;
    private int follower;

    public User(int mid, String name, int following, int follower) {
        this.mid = mid;
        this.name = name;
        this.following = following;
        this.follower = follower;
    }

    public int getMid() {
        return mid;
    }

    public void setMid(int mid) {
        this.mid = mid;
    }

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

    public int getFollowing() {
        return following;
    }

    public void setFollowing(int following) {
        this.following = following;
    }

    public int getFollower() {
        return follower;
    }

    public void setFollower(int follower) {
        this.follower = follower;
    }

    @Override
    public String toString() {
        return "User{" +
                "mid=" + mid +
                ", name='" + name + '\'' +
                ", following=" + following +
                ", follower=" + follower +
                '}';
    }
}

主体程序以上。



为增加运行效率可采取多线程

//线程
public class bilibiliThread implements Runnable{
    int mid=1;
    Object obj = new Object();
    @Override
    public void run() {
        User[] user=new User[100];
        while(mid<703200000){
            synchronized (obj){
                for(int i = 0 ;i<100;i++){
                    user[i] = MidUtil.getUser(mid);
                    //数据输出以便监控
                    System.out.println(user[i]);
                    mid++;
                }
                MidUtil.addToDatabase(user,"user");
            }
        }
        
    }
}

主程序

public class Test {
	//线程数
 	int ThreadNumber = 4;
    public static void main(String[] args) {
        bilibiliThread bt = new bilibiliThread();
        Thread[] threads =new Thread[ThreadNumber ];
        for (int i =0;i<ThreadNumber ;i++){
            threads[i] =new Thread(bt,"线程"+(i+1));
            threads[i].start();
        }
    }
}

给大家一些抓到的b站的ＡＰＩ
//用户名　会员状态　简介　头像　风纪委员状态等　
https://api.bilibili.com/x/space/acc/info?mid=UID
//粉丝数 关注数　等
https://api.bilibili.com/x/relation/stat?vmid=UID
//总观看量 总获赞数　文章获赞数等
https://api.bilibili.com/x/space/upstat?mid=UID
//视频详细数据
https://api.bilibili.com/x/web-interface/archive/stat?aid=AV号
https://api.bilibili.com/x/web-interface/view?bvid=BV号
//视频评论
http://space.bilibili.com/ajax/member/getSubmitVideos?mid=AV号&;pagesize=单页显示数&page=页数


https://www.bilibili.com/robots.txt
b站允许爬取的页面，建议遵守协议

(博主已经被b站屏蔽IP了)