一个简单词法分析器的实现代码（java实现）

               

http://www.cnblogs.com/xuqiang/archive/2010/09/21/1953501.html

Main.java

/* * 主程序 */import java.io.*;import lexer.*;public class Main { public static void main(String[] args) throws IOException {  Lexer lexer = new Lexer();    while (lexer.getReaderState() == false) {   lexer.scan();  }    /* 保存相关信息 */  lexer.saveTokens();  lexer.saveSymbolsTable();   }}

Lexer.java

package lexer;import java.io.*;import java.util.*;import symbols.*;public class Lexer { public static int line = 1;  /* 记录行号 */ char peek = ' ';  /* 下一个读入字符 */ Hashtable<String, Word> words =   new Hashtable<String, Word>(); /* 符号表 */ private Hashtable<Token, String> table =   new Hashtable<Token, String>(); /* token序列 */ private List<String> tokens =   new LinkedList<String> (); /* 读取文件变量 */ BufferedReader reader = null;  /* 保存当前是否读取到了文件的结尾  */ private Boolean isEnd = false;  /* 是否读取到文件的结尾 */ public Boolean getReaderState() {  return this.isEnd; }  /* 保存存储在table中的 */ public void saveSymbolsTable() throws IOException {  FileWriter writer = new FileWriter("符号表.txt");  writer.write("[符号]   [符号类型信息]\n");  writer.write("\r\n");    Enumeration<Token> e = table.keys();  while( e.hasMoreElements() ){   Token token = (Token)e.nextElement();   String desc = table.get(token);      /* 写入文件 */   writer.write(token + "\t\t\t" + desc + "\r\n");  }     writer.flush(); }  /* 保存Tokens */ public void saveTokens() throws IOException {  FileWriter writer = new FileWriter("Tokens表.txt");  writer.write("[符号] \n");  writer.write("\r\n");    for(int i = 0; i < tokens.size(); ++i) {   String tok = (String)tokens.get(i);      /* 写入文件 */   writer.write(tok + "\r\n");  }      writer.flush(); }  void reserve(Word w) {  words.put(w.lexme, w); }  /*  * 构造函数中将关键字和类型添加到hashtable words中  */ public Lexer() {  /* 初始化读取文件变量 */  try {   reader = new BufferedReader(new FileReader("输入.txt"));  }  catch(IOException e) {   System.out.print(e);  }      /* 关键字 */  this.reserve(new Word("if", Tag.IF));  this.reserve(new Word("then", Tag.THEN));  this.reserve(new Word("else", Tag.ELSE));  this.reserve(new Word("while", Tag.WHILE));  this.reserve(new Word("do", Tag.DO));    /* 类型 */  this.reserve(Word.True);  this.reserve(Word.False);  this.reserve(Type.Int);  this.reserve(Type.Char);  this.reserve(Type.Bool);  this.reserve(Type.Float); }  public void readch() throws IOException {  /* 这里应该是使用的是 */  peek = (char)reader.read();  if((int)peek == 0xffff){   this.isEnd = true;  }  // peek = (char)System.in.read(); }  public Boolean readch(char ch) throws IOException {  readch();  if (this.peek != ch) {   return false;  }    this.peek = ' ';  return true; }  public Token scan() throws IOException {  /* 消除空白 */   for( ; ; readch() ) {   if(peek == ' ' || peek == '\t')    continue;   else if (peek == '\n')     line = line + 1;   else    break;  }    /* 下面开始分割关键字，标识符等信息  */  switch (peek) {  /* 对于 ==, >=, <=, !=的区分使用状态机实现 */  case '=' :   if (readch('=')) {    tokens.add("==");    return Word.eq;    }   else {    tokens.add("=");    return new Token('=');   }  case '>' :   if (readch('=')) {    tokens.add(">=");    return Word.ge;   }   else {    tokens.add(">");    return new Token('>');   }  case '<' :   if (readch('=')) {    tokens.add("<=");    return Word.le;   }   else {    tokens.add("<");    return new Token('<');   }  case '!' :   if (readch('=')) {    tokens.add("!=");    return Word.ne;   }   else {    tokens.add("!");    return new Token('!');   }   }    /* 下面是对数字的识别，根据文法的规定的话，这里的   * 数字只要是能够识别整数就行.   */  if(Character.isDigit(peek)) {   int value = 0;   do {    value = 10 * value + Character.digit(peek, 10);    readch();   } while (Character.isDigit(peek));      Num n = new Num(value);   tokens.add(n.toString());   //table.put(n, "Num");   return n;  }    /*   * 关键字或者是标识符的识别   */  if(Character.isLetter(peek)) {   StringBuffer sb = new StringBuffer();      /* 首先得到整个的一个分割 */   do {    sb.append(peek);    readch();   } while (Character.isLetterOrDigit(peek));      /* 判断是关键字还是标识符 */   String s = sb.toString();   Word w = (Word)words.get(s);      /* 如果是关键字或者是类型的话，w不应该是空的 */   if(w != null) {    // table.put(w, "KeyWord or Type");    tokens.add(w.toString());    return w; /* 说明是关键字 或者是类型名 */   }      /* 否则就是一个标识符id */   w = new Word(s, Tag.ID);   tokens.add(w.toString());   table.put(w, "id");   words.put(s,  w);      return w;  }    /* peek中的任意字符都被认为是词法单元返回 */  Token tok  = new Token(peek);  // table.put(tok, "Token or Seprator");  if ((int)peek != 0xffff )    tokens.add(tok.toString());  peek = ' ';    return tok; }}

Num.java

package lexer;public class Num extends Token{ public final int value;  public Num(int v) {  super(Tag.NUM);  this.value = v; }  public String toString() {  return  "" + value; }}

Tag.java

package lexer;public class Tag { public final static int  AND  = 256,  BASIC = 257,  BREAK = 258,  DO  = 259,  ELSE = 260,  EQ  = 261, /* == */  FALSE = 262,  GE  = 263,  ID  = 264,  IF  = 265,  INDEX = 266,  LE  = 267,  MINUS = 268,  NE  = 269,  NUM  = 270,  OR  = 271,  REAL = 272,  TEMP = 273,  TRUE = 274,  WHILE = 275,  /* 后面添加 */  THEN = 276;}

Token.java

package lexer;public class Token { public final int tag;  public Token(int t) {  this.tag = t; }  public String toString() {  return "" + (char)tag; }  public static void main(String[] args) {  Token tok = new Token('a');  System.out.println(tok); }}

Word.java

/* * 类word用于管理保留字，标识符以及像&&这样的复合单词元素 。 */package lexer;public class Word extends Token { public String lexme = "";  public Word (String s, int t) {  super(t);  this.lexme = s; }  public String toString() {  return this.lexme; }  public static final Word   and = new Word("&&", Tag.AND),  or = new Word("||", Tag.OR),  eq = new Word ("==", Tag.EQ),  ne = new Word("!=", Tag.NE),  le = new Word("<=", Tag.LE),  ge = new Word(">=", Tag.GE),  minus = new Word("minus", Tag.MINUS),  True = new Word("true", Tag.TRUE),  False = new Word("false", Tag.FALSE),  temp = new Word("t", Tag.TEMP);}

Type.java

/* * 说明数据类型 */package symbols;import lexer.*;public class Type extends Word{ public Type(String s, int tag) {  super(s, tag); }  public static final Type  Int = new Type("int", Tag.BASIC),  Float = new Type("float", Tag.BASIC),  Char = new Type ("char", Tag.BASIC),  Bool =  new Type("bool", Tag.BASIC); }

 

＝＝＝＝＝＝＝＝＝＝＝＝

http://freewxy.iteye.com/blog/870016

什么是词法？ 

 

  所谓词法，源代码由字符流组成，字符流中包括关键字，变量名，方法名，括号等等符号，其中变量名要满足不能包括标点符号，不能以数字开头的数字与字母的字符串这个条件，对于括号要成对出现等等，这就是词法;

 

什么是词法分析？

 

  词法分析阶段是编译过程的第一个阶段。这个阶段的任务是从左到右一个字符一个字符地读入源程序，即对构成源程序的字符流进行扫描然后根据构词规则识别单词