libsvm 中文文本分类 java版本

这周打算用word2vec+lstm做一个中文文本分类模型，无奈老大以前用过libsvm，叫我用libsvm，折腾了两天基本上调通

中通碰到各种各样的问题，在此记录下来。

首先下载libsvm包，下载链接

http://www.csie.ntu.edu.tw/~cjlin/cgi-bin/libsvm.cgi?+http://www.csie.ntu.edu.tw/~cjlin/libsvm+zip  libsvm下载工具，下载之后解压 进入目录直接make命令既可以

然后把文本数据规范成如下格式：

2 2017:1.23527900896424 2080:1.3228803416955244 21233:3.475992040593523 
2 576:1.0467435856485432 967:1.0968877798239958 3940:1.7482714392181495 4449:1.7535719911308003 
2 967:1.0968877798239958 1336:1.3551722790297116 5611:1.8303003497257173 14735:1.7682821161365336 
1 7:0.02425295226485008 32:0.009012036411194203 80:0.0057407001135544745 127:0.020374370371014396 

标准的libsvm格式,分词用的是ansj工具，转化数值是tf-idf格式，其中特征的索引一定要按顺序排序，否则用libsvm工具训练的时候会爆如下错误：

Libsvm : Wrong input format at line 1

具体使用可以参考这篇博客：http://endual.iteye.com/blog/1267442，关键是要知道怎么生成libsvm格式文件，这个是关键。

下面贴上把文本转化为libsvm的格式工具的代码，用了许多1.8的特性，习惯了写scala，突然用java感觉很繁琐，见谅：

package com.meituan.model.libsvm;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;

import org.ansj.splitWord.analysis.ToAnalysis;
import org.apache.commons.lang3.StringUtils;

import com.meituan.nlp.util.WordUtil;
import com.meituan.nlp.util.TextUtil;
import com.meituan.model.util.Config;

public class DocumentTransForm {
	private static String inputpath = Config.getString("data.path");
	private static String outputpath = Config.getString("data.libsvm");
	private static Map<String, Terms> mapTerms = new HashMap<String, Terms>();
	public static int documentTotal = 0;

	public static void getTerms(String file) {
		BufferedReader br = null;
		try {
			br = new BufferedReader(new InputStreamReader(new FileInputStream(
					file)));

			String lines = br.readLine();
			int featurecount = 1;
			while (lines != null) {
				String line = lines.split("\t")[0];
				Set<String> sets = ToAnalysis
						.parse(WordUtil.replaceAllSynonyms(TextUtil
								.fan2Jian(WordUtil.replaceAll(line
										.toLowerCase()))))
						.getTerms()
						.stream()
						.map(x -> x.getName())
						.filter(x -> !WordUtil.isStopword(x) && x.length() > 1
								&& !WordUtil.startWithNumeber(x))
						.collect(Collectors.toSet());
				if (sets != null) {

					for (String key : sets) {
						if (!mapTerms.containsKey(key)) {
							Terms terms = new Terms(key, featurecount);
							mapTerms.put(key, terms);
							featurecount++;
						} else {
							mapTerms.get(key).incrFreq();
						}
					}
					documentTotal++;
				}

				lines = br.readLine();

			}

		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			if (br != null) {
				try {
					br.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
	}

	public static void getLibsvmFile(String input, String output) {
		BufferedReader br = null;
		BufferedWriter bw = null;

		try {
			br = new BufferedReader(new InputStreamReader(new FileInputStream(
					input)));

			bw = new BufferedWriter(new OutputStreamWriter(
					new FileOutputStream(output)));

			String lines = br.readLine();

			while (StringUtils.isNoneBlank(lines)) {
				String label = lines.split("\t")[1].equalsIgnoreCase("-1") ? "2"
						: "1";
				String content = lines.split("\t")[0];
				Map<String, Long> maps = ToAnalysis
						.parse(WordUtil.replaceAllSynonyms(TextUtil
								.fan2Jian(WordUtil.replaceAll(content
										.toLowerCase()))))
						.getTerms()
						.stream()
						.map(x -> x.getName())
						.filter(x -> !WordUtil.isStopword(x) && x.length() > 1
								&& !WordUtil.startWithNumeber(x))
						.collect(
								Collectors.groupingBy(p -> p,
										Collectors.counting()));

				if (maps != null && maps.size() > 0) {
					StringBuffer sb = new StringBuffer();
					sb.append(label).append(" ");
					int sum = maps
							.values()
							.stream()
							.reduce((result, element) -> result = result
									+ element).get().intValue();

					Map<Integer, Double> treeMap = new TreeMap<>();
					for (Entry<String, Long> map : maps.entrySet()) {

						String key = map.getKey();
						double tf = TFIDF.tf(map.getValue(), sum);
						// 这个key一定存在
						double idf = TFIDF.idf(documentTotal, mapTerms.get(key)
								.getFreq());

						treeMap.put(mapTerms.get(key).getId(),
								TFIDF.tfidf(tf, idf));

					}
					treeMap.forEach((x, y) -> sb.append(x).append(":")
							.append(y).append(" "));
					bw.write(sb.toString());
					bw.newLine();
				}

				lines = br.readLine();

			}

		} catch (Exception e) {
			e.printStackTrace();

		} finally {
			try {
				bw.close();
				br.close();

			} catch (Exception e) {
				e.printStackTrace();
			}
		}

	}

	public static void main(String[] args) {

		getTerms(inputpath);
		System.out.println("documentTotal is :" + documentTotal);
		getLibsvmFile(inputpath, outputpath);

		List<String> list = new ArrayList<String>(Arrays.asList("a", "a"));
		Map<String, Long> map = list.stream().collect(
				Collectors.groupingBy(p -> p, Collectors.counting()));
		System.out.println(map.values().stream()
				.reduce((result, element) -> result = result + element).get()
				.intValue());

	}

}

然后开始训练，首先对数据标准化:

./svm-scale  -l 0 -u 1  /Users/shuubiasahi/Documents/workspace/spark-model/file/libsvem.txt >/Users/shuubiasahi/Documents/workspace/spark-model/file/libsvem_scale.txt

开始训练，libsvm提供了若干的参数 ，运行./svm-train，可以看到

./svm-train -h 0 -t 0 /Users/shuubiasahi/Documents/workspace/spark-model/file/libsvem_scale.txt /Users/shuubiasahi/Documents/workspace/spark-model/file/model.txt

svm的理论我个人认为还是比较简单，可以看李航老师那本统计学习方法，一看就明白。