数据挖掘关于分类方面

1.提供自动学习数据，对这些数据进行索引
2。从索引里提取一个2维矩阵列表，写入一个文本文件里。关键性代码如下：

java 代码

 

1. import java.io.File;  
2. import java.util.HashMap;  
3.   
4. import org.apache.log4j.Logger;  
5. import org.apache.lucene.document.Document;  
6. import org.apache.lucene.index.IndexReader;  
7. import org.apache.lucene.index.Term;  
8. import org.apache.lucene.index.TermEnum;  
9. import org.apache.lucene.search.BooleanQuery;  
10. import org.apache.lucene.search.Hits;  
11. import org.apache.lucene.search.IndexSearcher;  
12. import org.apache.lucene.search.Query;  
13. import org.apache.lucene.search.TermQuery;  
14. import org.apache.lucene.search.BooleanClause.Occur;  
15. import org.apache.lucene.store.Directory;  
16. import org.apache.lucene.store.FSDirectory;  
17.   
18. import com.feedsky.classify.util.FileUtil;  
19.   
20. public class MMClassify{  
21.       
22.     private static String indexDir = "D:\\projectsvn\\trunk\\classify\\indexFile";//索引目录  
23.     private static String resultFile = "D:\\projectsvn\\trunk\\classify\\resultFile.txt";  
24.     private  IndexSearcher searcher ;  
25.     private  IndexReader reader;  
26.     private Logger logger = Logger.getLogger(MMClassify.class);  
27.     private HashMap clazzMap = new HashMap();  
28.     private String[] clazzArray = {"C000007","C000008","C000010","C000013","C000014","C000016","C000020","C000022","C000023","C000024"};  
29.   
30.       
31.     public static void main(String[] args) {  
32. //      Matrix matrix = new Matrix(10,2);  
33. //      matrix.set(0, 1, 0.5);  
34. //      matrix.set(1, 1, 0.5);  
35. //  //  System.out.println(matrix);  
36.         MMClassify mm = new MMClassify();  
37.         mm.init();  
38.         mm.process();  
39.         // System.out.println(mm.matrix);  
40.     }  
41.       
42.     public void init(){  
43.         try{  
44.             Directory directory = FSDirectory.getDirectory(indexDir);  
45.             reader = IndexReader.open(directory);  
46.             searcher = new IndexSearcher(reader);  
47.               
48.              
49.             clazzMap.put("C000007", "汽车");  
50.             clazzMap.put("C000008", "财经");  
51.             clazzMap.put("C000010", "IT");    
52.             clazzMap.put("C000013", "健康");  
53.             clazzMap.put("C000014", "体育");  
54.             clazzMap.put("C000016", "旅游");  
55.             clazzMap.put("C000020", "教育");  
56.             clazzMap.put("C000022", "招聘");  
57.             clazzMap.put("C000023", "文化");  
58.             clazzMap.put("C000024", "军事");  
59.      
60.         }catch(Exception e){  
61.             e.printStackTrace();  
62.         }  
63.     }  
64.       
65.     public void process(){  
66.         try{  
67.             //提取所有关键词的Term集合  
68.             TermEnum terms = reader.terms();  
69.             while (terms.next()) {  
70.                 //提取其中一个Term  
71.                 Term term = terms.term();  
72.                 String keyWord = term.text();  
73.                 //判断该关键词的所属域  
74.                 if(term.field().equals("itemContent")){  
75.                     BooleanQuery booleanQuery = new BooleanQuery();  
76.                     Query keywordQuery = new TermQuery(new Term("itemContent",keyWord));  
77.                     booleanQuery.add(keywordQuery,Occur.MUST);  
78.                     //到索引文件里做全部搜索  
79.                     Hits hits = searcher.search(keywordQuery);  
80.                       
81.                     ResultList list = new ResultList(keyWord);  
82.                     for (int i = 0; i < hits.length(); i++) {  
83.                         Document doc = (Document)hits.doc(i);  
84.                         float score = hits.score(i);  
85.                         //doc.get("dirname");获得分类编号  
86.                         Result result = new Result(score,doc.get("dirname"));  
87.                         list.addResult(result);  
88.                     }  
89.                     HashMap map = list.resultMap;  
90.                       
91.                     //提取二维矩阵表  
92.                     StringBuffer resultStr = new StringBuffer();  
93.                      
94.                     for(int i=0;i
95.                         String clazz = clazzArray[i];  
96.                         if(map.get(clazz) == null){  
97.                             resultStr.append(0 + "  ");  
98.                         }else{  
99. //                          Float score = (Float)map.get(clazz);  
100. //                          resultStr.append(score.floatValue() + "  ");  
101.                             resultStr.append(list.getValue(clazz) + "  ");  
102.                         }  
103.                     }  
104. //                  System.out.println(resultStr.toString());  
105.                     Boolean isAppend = true;  
106.                     File file = new File(resultFile);  
107.                     if(!file.isFile()){  
108.                         isAppend = false;  
109.                     }  
110.                     String content = keyWord + "  " + resultStr.toString();  
111.                     FileUtil.writeFileByLine(content , resultFile , "GBK" , isAppend);  
112.                       
113.                     //提取该关键词在所有分类里的分值  
114. //                  Iterator it = map.keySet().iterator();  
115. //                  while (it.hasNext()) {  
116. //                      // Get key,分类编号  
117. //                      String key = (String)it.next();  
118. //                      Float score = (Float)map.get(key);  
119. //                       
120. //                      //该关键词所在的分类以及分值  
121. //                      String outline = keyWord + " clazz:" + (String)clazzMap.get(key) + "  score:" + score.floatValue();  
122. //                      System.out.println(outline);  
123. //                      logger.info(outline);  
124. //                        
125. //                  }  
126.                       
127.                       
128.     //              matrix = new Matrix(hits.length(),2);  
129.                 }  
130.                   
131.                  //FileUtil.writeFileByLine(out.toString(), "D:\\projectsvn\\trunk\\classify\\out.txt", "gbk", true);  
132.                   
133.             }  
134.         }catch(Exception e){  
135.             e.printStackTrace();  
136.         }  
137.     }  
138.   
139. }  
140.   
141.   
142. class Result {  
143.     public Float score;  
144.     public String clazz;  
145.       
146.     public Result(float score, String clazz) {  
147.         super();  
148.         this.score = score;  
149.         this.clazz = clazz;  
150.     }  
151. }  
152.   
153.   
154. class ResultList{  
155.     public HashMap resultMap = new HashMap();  
156.     public String keyword ;  
157.       
158.     public ResultList(String keyword) {  
159.         super();  
160.         this.keyword = keyword;  
161.     }  
162.   
163.     public void addResult(Result result){  
164.         Object score = resultMap.get(result.clazz);  
165.         if(score !=null ){  
166.             float scorez =(Float)score;  
167.             score = scorez + result.score;  
168.             resultMap.put(result.clazz, score);  
169.         }else{  
170.             resultMap.put(result.clazz, result.score);  
171.         }  
172.     }  
173.       
174.     public Float getValue(String clazz){  
175.           
176.         Object score = resultMap.get(clazz);  
177.         if(score!=null){  
178.             float countScore = (float)0.000000001;  
179.             Iterator it = resultMap.keySet().iterator();  
180.             while (it.hasNext()) {  
181.                 // Get key  
182.                 Object key = it.next();  
183.                 Float sc = (Float)resultMap.get(key);  
184.                 countScore += sc;  
185.             }  
186.               
187.             float scoreAve=(Float)score/countScore;  
188.             return scoreAve;  
189.         }  
190.           
191.         return (float)0;  
192.     }  
193.       
194.     public Float getHot(String clazz){  
195.           
196.         Object score = resultMap.get(clazz);  
197.         if(score!=null){  
198.             return (Float)score;  
199.         }  
200.           
201.         return (float)0;  
202.     }  
203.       
204. }