抓取腾讯新闻评论

最新推荐文章于 2024-02-01 18:39:54 发布

大橙子真帅

最新推荐文章于 2024-02-01 18:39:54 发布

阅读量2k

点赞数

分类专栏：数据抓取/爬虫文章标签：爬虫抓取腾讯新闻评论

本文链接：https://blog.csdn.net/itjavaer/article/details/49074875

版权

数据抓取/爬虫专栏收录该内容

7 篇文章 0 订阅

订阅专栏

package com.orange.qqnews;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import net.sf.json.JSONObject;

public class Test {
	public static void main(String[] args) {
		//新闻正文正则
		String regex1 = "<div id=\"Cnt-Main-Article-QQ\" bossZone=\"content\">([\\d\\D]*?)</div>";
		//评论ID正则
		String regex2 = "cmt_id = ([\\d]*?);";
		//获取网页源代码
		String html = openUrl("http://news.qq.com/a/20150825/004734.htm","gb2312");
		//获取新闻正则
		String content = getContent(regex1,html);
		System.out.println(content);
		//获取评论ID
		String cmtId = getContent(regex2,html);
		System.out.println(cmtId);
		
		//拼接评论地址
		String cmtUrl = "http://coral.qq.com/article/"+cmtId+"/comment?commentid=0&reqnum=20";
		String cmt = openUrl(cmtUrl,"gb2312");

		JSONObject jsonMap = new JSONObject();
		Map map = jsonMap.fromObject(cmt);
		Map<String,List> data = (Map)map.get("data");
		List<Map<String,String>> comments = data.get("commentid");
		
		for(Map<String,String> m : comments){
			String cmtContent = m.get("content"); //评论
			
			//其他信息略过(回复人，回复时间，赞等)
			
			System.out.println(cmtContent);
		}
		

	}

	/**
	 * 访问url返回url的html代码
	 */
	public static String openUrl(String currentUrl,String charset) {
		InputStream is = null;
		BufferedReader br = null;
		URL url;
		StringBuffer html = new StringBuffer();
		try {
			url = new URL(currentUrl);
			URLConnection conn = url.openConnection();
			conn.setReadTimeout(5000);
			conn.connect();
			is = conn.getInputStream();
			br = new BufferedReader(new InputStreamReader(is,charset));
			String str;
			while (null != (str = br.readLine())) {
				html.append(str).append("\n");
			}
		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			if (br != null) {
				try {
					br.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
			if (is != null) {
				try {
					is.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}

		}
		return html.toString();
	}
	
	private static String getContent(String regex,String text) {
		String content = "";
		Pattern pattern = Pattern.compile(regex);
		Matcher matcher = pattern.matcher(text);
		while(matcher.find()) {
			content = matcher.group(1).toString();
		}
		return content;
	}
}