爬取京东评论、分词+词频统计、词云图展示

一、爬取京东评论

京东评论竟然全部对外开放

public class CommentCrawler {
    final static PoolingHttpClientConnectionManager httpClientConnectionManager = new PoolingHttpClientConnectionManager();
    final static int MAX_PAGE = 50;

    static HttpClient getClient() {
        return HttpClients.custom().setConnectionManager(httpClientConnectionManager).build();
    }

    static String getUrl(String productId, int page) {
        return String.format(
                "http://sclub.jd.com/comment/productPageComments.action?productId=%s&score=0&sortType=3&page=%d&pageSize=10",
                productId, page);
    }

    static Comment commentFromJson(JSONObject json, String productId) {
        return new Comment(json.getLongValue("id"), productId, json.getString("score"), json.getString("content"));
    }

    public static boolean crawlComments(String productId) {
        try {
            int maxPage = 1;
            int nowPage = 0;
            HttpClient client = getClient();
            while (nowPage < maxPage) {
                String url = getUrl(productId, nowPage);
                HttpGet get = new HttpGet(url);
                HttpResponse resp = client.execute(get);
                JSONObject json = JSON.parseObject(EntityUtils.toString(resp.getEntity()));
                JSONArray comments = json.getJSONArray("comments");
                if (comments.size() == 0)
                    return false;
                CommentService ser = new CommentService();
                for (int i = 0; i < comments.size(); i++) {
                    Comment comment = commentFromJson(comments.getJSONObject(i), productId);
                    ser.insertComment(comment);
                }
                if (nowPage == 0) {
                    maxPage = json.getInteger("maxPage");
                    ser.insertProduct(new Product(productId, comments.getJSONObject(0).getString("referenceName")));
                }
                nowPage++;
            }
            ProductJudger.judge(productId);
            return true;
        } catch (Exception e) {
            e.printStackTrace();
        }
        return false;
    } 
}

二、结巴分词

jieba分词原本是Python版的，有人把它改成了Java版，名字也改成了jieba-analysis

JiebaSegmenter segmenter = new JiebaSegmenter();
        List<Word> list = segmenter.sentenceProcess(str);
        for (Word i : list) {
            String token = i.getToken();
            if (stopWords.contains(token)) {
                continue;
            }
            Integer cnt = map.get(token);
            if (cnt == null) {
                cnt = 0;
            }
            map.put(token, new Integer(cnt + 1));
        }

三、词云图

用到d3.js，d3-cloud.js这两个库，d3.js是“Data Driven Document”，d3-cloud这个库还是比较难用的，主要是官方实例代码太少了。

这里给出一个例子：每一个词云图都对应一个字典，这个字典就是“词语：频率”这样的键值对。给定多个字典，每一个字典都要渲染成一个词云图。

<div id="word-clouds" style="text-align: center"></div>
<script>
    var wordClouds = <%=request.getAttribute("wordClouds")%>;
    $(document).ready(function () {
        for (var i in wordClouds) {
            var it = wordClouds[i];
            var divId = "product_wordcloud" + it['productId'];
            $("#word-clouds").append("<h3 align='center'>商品" + it['productId'] + "词云图</h3>")
                    .append("<div id='" + divId + "'></div>");
            createWordCloud(transformWordFraquency(it['words']), "#" + divId)
        }
    })
</script>

还需要编写如下JS代码

var fill = d3.scale.category20();// 20种颜色
var wordCloudWidth = 800, wordCloudHeight = 400;
var font_name = "楷体", font_weight = "bold", max_font_size = 50;
var word_count = 50;// 显示词汇个数
var word_max_size = 60;// 显示词汇字体最大字号
var word_min_size = 10;// 显示词汇字体最小字号
/*
 * 函数名称：transformWordFrequency
 * 参数words：字典类型，形如“{word1:cnt1,word2:cnt2,word3:cnt3}” 返回值：{text，size}数组
 */

function transformWordFraquency(words) {
    var ar = []
    for ( var i in words) {
        ar.push({
            "text" : i,
            "size" : words[i]
        })
    }
    // 按照字体的大小从大到小进行排序，只取出现次数较多的前几名
    ar.sort(function(x, y) {
        return y['size'] - x['size'];
    })
    ar = ar.slice(0, Math.min(word_count, ar.length));
    for (var i = 0; i < ar.length; i++) {
        ar[i]['size'] = word_max_size - (word_max_size - word_min_size)
                / ar.length * i;
    }
    return ar;
}
/*
 * wordMap是[{text:"",size:""}]形式的数组 selector是即将渲染到的目标位置
 */
function createWordCloud(wordMap, selector) {
    d3.layout.cloud().size(
            [ wordCloudWidth * 2 - 100, wordCloudHeight * 2 - 100 ]).words(
            wordMap).font(font_name).fontWeight(font_weight).fontSize(
            function(d) {
                return d.size;
            }).rotate(function() {
        return 0;
    }).on("end", function(words) {
        renderWordCloud(words, selector)
    }).start();
}
/*
 * 像这种风格的函数调用，这些函数的调用顺序不能变
 */
function renderWordCloud(words, selector) {
    d3.select(selector).append("svg").attr("width", wordCloudWidth).attr(
            "height", wordCloudHeight).append("g")
            .attr(
                    "transform",
                    "translate(" + wordCloudWidth / 2 + "," + wordCloudHeight
                            / 2 + ")").selectAll("text").data(words).enter()// 进入words，相当于for循环
            .append("text").style("font-family", font_name).style(
                    "font-weight", font_weight)
            // .attr("text-anchor", "middle")
            .style("font-size", function(d) {// 字体大小
                return d.size + "px";
            }).style("fill", function(d, i) {// 字体颜色
                return fill(i);
            }).attr(
                    "transform",
                    function(d) {
                        return "translate(" + [ d.x, d.y ] + ") rotate("
                                + d.rotate + ")";
                    }).text(function(d) {
                return d.text;
            });
}