java实现文本分类中卡方特征选择

java在文本分类中卡方的特征选择，

 在文本分类的特征选择阶段，一般使用“词汇t与类别c不相关”来做原假设，计算出的开方值越大，说明对原假设的偏离越大，我们越倾向于认为原

假设的反面情况是正确的。选择的过程为每个词计算它与类别c的开方值，从大到小排个序（此时开方值越大越相关），取前k个就可以。所以卡方值越大，

词汇与分类越相关。

最后结果，一个特征对应一个最后的卡方值：

挺 81613.22937057534
特别 80715.8079021637
太 77021.62013902201
按摩 77012.332216897
喜欢 74483.97317313829
好吃 68723.25474753922
推荐 64809.29378047458
朋友 55988.55238512035
里 55371.21236337274
舒服 54673.57763700053
服务员 53918.5463140205
手法 53379.51974620014
体验 52445.83665612151

看代码实现:

package com.meituan.model.libsvm;

public class ChiSquare {
	private String word;
	private double A;
	private double B;
	private double C;
	private double D;
	public double chisq;

	public ChiSquare() {

	}

	public ChiSquare(String word) {
		this.word = word;
	}

	public ChiSquare(String word, int a, int b, int c, int d) {
		super();
		this.word = word;
		this.A = a;
		this.B = b;
		this.C = c;
		this.D = d;
		this.chisq = getChisq();
	}

	public String getWord() {
		return word;
	}

	public void setWord(String word) {
		this.word = word;
	}

	public double getA() {
		return A;
	}

	public void setA(int a) {
		this.A = a;
	}

	public double getB() {
		return B;
	}

	public void setB(int b) {
		this.B = b;
	}

	public double getC() {
		return C;
	}

	public void setC(int c) {
		this.C = c;
	}

	public double getD() {
		return D;
	}

	public void setD(int d) {
		this.D = d;
	}

	public void incrA() {
		A = A + 1;
	}

	public void incrB() {
		B++;
	}

	public void decrementC() {
		C--;
	}

	public void decrementD() {
		D--;
	}

	public void setChisq() {
		this.chisq = getChisq();
	}

	public double getChisq() {
		if (A == 0 && B == 0 && C == 0 && D == 0) {
			return 0;
		} else {
			return Math.pow((A * D - B * C) * 1.0, 2) * 155738.0
					/ ((A + C) * (A + B) * (B + D) * (C + D));
		}
	}

	@Override
	public String toString() {
		return String.format(
				"word=%s  ChiSquare=%f [A=%f , B=%f , C=%f , D=%f  ]",
				this.word, this.getChisq(), this.A, this.B, this.C, this.D);
	}

	public static void main(String[] args) {
		ChiSquare chiSquare = new ChiSquare("搀扶", 17, 1, 20936, 134784);
		System.out.println(chiSquare.toString());
	}
}

package com.meituan.model.libsvm;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.stream.Collectors;

import org.ansj.splitWord.analysis.ToAnalysis;

import com.meituan.model.util.Config;
import com.meituan.nlp.util.TextUtil;
import com.meituan.nlp.util.WordUtil;

public class ChisQuareSelect {
	private static String inputpath = Config.getString("data.path");
	private static Map<String, ChiSquare> mapchis = new HashMap<String, ChiSquare>();

	public static void main(String[] args) {
		System.setProperty("java.util.Arrays.useLegacyMergeSort", "true");
		int[] arr = getTotal(inputpath);
		System.out.println("arr is :"+Arrays.toString(arr));
		init(inputpath, arr);
		featureselect(inputpath);

		System.out.println("排序开始");

		List<Map.Entry<String, ChiSquare>> list = new ArrayList<Map.Entry<String, ChiSquare>>(
				mapchis.entrySet());

		Collections.sort(list, new Comparator<Map.Entry<String, ChiSquare>>() {

			@Override
			public int compare(Entry<String, ChiSquare> o1,
					Entry<String, ChiSquare> o2) {
				double result = o1.getValue().getChisq()
						- o2.getValue().getChisq();
				if (result > 0) {
					return -1;
				}else{
				
				return 1;
				}
			}
		});

		write(list, "file/chisq.txt", arr);
		System.out.println("success");

	}

	public static void write(List<Map.Entry<String, ChiSquare>> list, String out, int[] arr) {
		BufferedWriter bw = null;
		int sum = 0;
		for (int ar : arr) {
			sum += ar;
		}
		System.out.println("sum is :"+sum);
		try {
			bw = new BufferedWriter(new OutputStreamWriter(
					new FileOutputStream(out)));
			for (Map.Entry<String, ChiSquare> map : list) {
				bw.write(map.getKey() + " " + map.getValue().getChisq() 
						+ "\n");
				if(Math.random()<0.001){
					System.out.println(map.getValue().toString());
				}
			}

		} catch (Exception e) {

			e.printStackTrace();

		} finally {
			if (bw != null) {
				try {
					bw.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}

	}

	public static int[] getTotal(String in) {
		BufferedReader br = null;

		int[] arr = new int[2];
		int totalP = 0;
		int totalN = 0;

		try {
			br = new BufferedReader(new InputStreamReader(new FileInputStream(
					in)));
			String lines = br.readLine();

			while (lines != null) {
				String label = lines.split("\t")[1].equalsIgnoreCase("-1") ? "-1"
						: "1";
				if (label.equals("-1")) {
					totalN++;
				} else {
					totalP++;
				}

				lines = br.readLine();

			}

		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			if (br != null) {
				try {
					br.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
		arr[0] = totalP;
		arr[1] = totalN;
		return arr;
	}

	public static void init(String in, int[] arr) {
		BufferedReader br = null;

		try {
			br = new BufferedReader(new InputStreamReader(new FileInputStream(
					in)));
			String lines = br.readLine();

			while (lines != null) {
				String content = lines.split("\t")[0];
				Set<String> sets = ToAnalysis
						.parse(WordUtil.replaceAllSynonyms(TextUtil
								.fan2Jian(WordUtil.replaceAll(content
										.toLowerCase()))))
						.getTerms()
						.stream()
						.map(x -> x.getName())
						.filter(x ->WordUtil.isChineseChar(x) &&  !WordUtil.isStopword(x)
								&& !WordUtil.startWithNumeber(x))
						.collect(Collectors.toSet());

				for (String s : sets) {

					if (!mapchis.containsKey(s)) {
						mapchis.put(s, new ChiSquare(s, 0, 0, arr[0], arr[1]));
					}

				}

				lines = br.readLine();

			}

		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			if (br != null) {
				try {
					br.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
	}

	public static void featureselect(String in) {
		BufferedReader br = null;

		try {
			br = new BufferedReader(new InputStreamReader(new FileInputStream(
					in)));
			String lines = br.readLine();

			while (lines != null) {
				String content = lines.split("\t")[0];
				String label = lines.split("\t")[1].equalsIgnoreCase("-1") ? "-1"
						: "1";

				Set<String> sets = ToAnalysis
						.parse(WordUtil.replaceAllSynonyms(TextUtil
								.fan2Jian(WordUtil.replaceAll(content
										.toLowerCase()))))
						.getTerms()
						.stream()
						.map(x -> x.getName())
						.filter(x -> WordUtil.isChineseChar(x) && !WordUtil.isStopword(x)
								&& !WordUtil.startWithNumeber(x))
						.collect(Collectors.toSet());

				for (String s : sets) {
					if ("-1".equalsIgnoreCase(label)) {
						mapchis.get(s).incrA();
						mapchis.get(s).decrementC();
					} else if ("1".equalsIgnoreCase(label)) {
						mapchis.get(s).incrB();
						mapchis.get(s).decrementD();
					}

				}

				lines = br.readLine();

			}

		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			if (br != null) {
				try {
					br.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
	}

}