import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.RandomAccessFile;
import java.util.StringTokenizer;
/*
* 文本格式:已分词的中文文本,空格切割。有若干行。每行为一个段落。
* 功能:遍历文档,逐个返回词语。
* 两种模式:
* 1 到文档末尾后,结束
* 2 到文档末尾后。从头再读。
/ public class WordReader { static final int normalMode = 0; static final int againMode = 1; int currentMode = 0; //BufferedReader br=null; RandomAccessFile raf= null; StringTokenizer tokenizer = null; String nextWord=null; int currentLine = 0; int allCounts = 0; public WordReader(String fileName) throws IOException { File file=new File(fileName); //br=new BufferedReader(new InputStreamReader(new FileInputStream(file),"utf-8")); raf = new RandomAccessFile(file,"r") ; } private boolean hasNextWord() throws IOException { if( tokenizer!=null && tokenizer.hasMoreTokens()) { nextWord = tokenizer.nextToken(); return true; } else { String line=raf.readLine(); if(line == null) { if(currentMode == normalMode) return false; else //从头再来 { raf.seek(0); return hasNextWord();//递归 } } tokenizer = null; line = new String(line.getBytes("iso8859-1"),"utf-8"); tokenizer= new StringTokenizer(line," "); return hasNextWord();//递归 } } private String getNextWord() throws IOException { if(nextWord != null) { String word = nextWord; nextWord = null; allCounts ++; return word; } else if(hasNextWord()) { return getNextWord(); } else return null; } public static void main(String[] args) throws IOException { // TODO Auto-generated method stub WordReader wordReader = new WordReader("/home/linger/sources/ParaModel/electronic_seg.txt"); wordReader.currentMode = WordReader.againMode; //while(wordReader.hasNextWord())//共10329309个词 for(int i=0;i<10329319;i++)//文本从头读 { System.out.println(wordReader.getNextWord()); } System.out.println(wordReader.allCounts); } }
用randomaccessfile类非常easy操作文件指针。
可是遇到中文乱码问题。參考了这里http://blog.chinaunix.net/uid-15490606-id-211958.html。攻克了。
line = new String(line.getBytes("iso8859-1"),"utf-8");
对编码不是非常精通。有时见看看这个http://blog.sina.com.cn/s/blog_673c81990100t1lc.html。