词法分析器JavaCC
程序输入:从test.txt文件中读取内容
void main()
{
while(a>45)
{
int x,x1;
x=x1+1;
}
}
程序输出:识别出的单词序列,格式为:(种别码,属性值)
一、支持类型
能识别关键字、标识符、常量(整常数、浮常数、字符串)、运算符、分隔符。能处理单行和多行注释。支持识别的各单词种别码及宏定义如下表
单词 | 种别码 | 宏 | 单词 | 种别码 | 宏 | 单词 | 种别码 | 宏 |
and | 1 | AND | Or | 22 | OR | / | 43 | RDIV |
array | 2 | ARRAY | Packed | 23 | PACKED | = | 44 | EQ |
begin | 3 | BEGIN | procedure | 24 | PROC | < | 45 | LT |
case | 4 | CASE | Program | 25 | PROG | > | 46 | GT |
const | 5 | CONST | Record | 26 | RECORD | <= | 47 | LE |
div | 6 | DIV | Repeat | 27 | REPEAT | >= | 48 | GE |
do | 7 | DO | Set | 28 | SET | <> | 49 | NE |
downto | 8 | DOWNTO | Then | 29 | THEN | ( | 50 | LA_BRAC |
else | 9 | ELSE | To | 30 | TO | ) | 51 | BR_BRAC |
end | 10 | END | Type | 31 | TYPE | , | 52 | COMMA |
file | 11 | FILE | Until | 32 | UNTIL | 、 | 53 | P_MARK |
for | 12 | FOR | Car | 33 | CAR | . | 54 | F_STOP |
function | 13 | FUNC | While | 34 | WHILE | .. | 55 | RANGE |
goto | 14 | GOTO | With | 35 | WITH | : | 56 | COLON |
if | 15 | IF | 标识符 | 36 | ID | := | 57 | ASSIGN |
in | 16 | IN | 整常数 | 37 | INT | ; | 58 | SEMIC |
label | 17 | LABEL | 实常数 | 38 | REAL | ^ | 59 | CAP |
mod | 18 | MOD | 字符串 | 39 | STRING | ** | 60 | EXP |
nil | 19 | NIL | + | 40 | PLUS | [ | 61 | LS_BRAC |
not | 20 | NOT | - | 41 | MINUS | ] | 62 | RS_BRAC |
of | 21 | OF | * |
| MULTI | ‘ | 63 | Q_MARK |
main方法
public class lexical
{
public static void main(String args [])
{
System.out.println("Reading from standard input...");
System.out.print("Enter an expression like \"1+(2+3)*var;\" :");
try {
lexical a;
String file = "test.txt";
FileInputStream fin=new FileInputStream(file);
SimpleCharStream scs = new SimpleCharStream(fin);
lexicalTokenManager tm=new lexicalTokenManager(scs);
Token t;
int i=0;
while(true)
{
t=tm.getNextToken();
if(t.kind==0)break;
System.out.println(t.kind+ "\t" +t.toString());
}
} catch (FileNotFoundException e) {
// TODO 自动生成的 catch 块
e.printStackTrace();
}
}
}
TOKEN
TOKEN : /* LITERALS */
{
< INTEGER_LITERAL :
< DECIMAL_LITERAL > ([ "l", "L" ])?
| < HEX_LITERAL > ([ "l", "L" ])?
| < OCTAL_LITERAL > ([ "l", "L" ])?
>
| < #DECIMAL_LITERAL : [ "1"-"9" ] ([ "0"-"9" ])* >
| < #HEX_LITERAL : "0" [ "x", "X" ] ([ "0"-"9", "a"-"f", "A"-"F" ])+ >
| < #OCTAL_LITERAL : "0" ([ "0"-"7" ])* >
}
TOKEN : /*keywords*/
{
< WHILE : "while" >
| < IF : "if" >
}
TOKEN : /*OPERATORS*/
{
< ADD : "+" >
| < MUL : "*" >
}
TOKEN : /*separators*/
{
< COMMA : "," >
| < SEMI : ";" >
}
TOKEN : /*kuohao*/
{
< LPAREN : "(" >
| < RPAREN : ")" >
| < LCURLY : "{" >
| < RCURLY : "}" >
| < GT : ">" >
| < EQ : "=" >
| < LT : "<" >
}
TOKEN : /* IDENTIFIERS */
{
< IDENTIFIER :
< LETTER >
(
< LETTER >
| < DIGIT >
)* >
| < #LETTER : [ "_", "a"-"z", "A"-"Z" ] >
| < #DIGIT : [ "0"-"9" ] >
}
lexical.java源代码
/* Generated By:JJTree&JavaCC: Do not edit this line. lexical.java */
package lexicalll;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
public class lexical/*@bgen(jjtree)*/implements lexicalTreeConstants, lexicalConstants {/*@bgen(jjtree)*/
protected JJTlexicalState jjtree = new JJTlexicalState();public static void main(String args [])
{
System.out.println("Reading from standard input...");
System.out.print("Enter an expression like \u005c"1+(2+3)*var;\u005c" :\n");
// lexical parser = new lexical(System.in);
// try
// {
// SimpleNode n = parser.Start();
// n.dump("");
// System.out.println("Thank you.");
// }
// catch (Exception e)
// {
// System.out.println("Oops.");
// System.out.println(e.getMessage());
// }
try {
lexical a;
String file = "test.txt";
FileInputStream fin=new FileInputStream(file);
SimpleCharStream scs = new SimpleCharStream(fin);
lexicalTokenManager tm=new lexicalTokenManager(scs);
Token t;
int i=0;
while(true)
{
t=tm.getNextToken();
if(t.kind==0)break;
System.out.println(t.kind+ "\u005ct" +t.toString());
}
} catch (FileNotFoundException e) {
// TODO 自动生成的 catch 块
e.printStackTrace();
}
}
final public SimpleNode Start() throws ParseException {
/*@bgen(jjtree) Start */
SimpleNode jjtn000 = new SimpleNode(JJTSTART);
boolean jjtc000 = true;
jjtree.openNodeScope(jjtn000);
try {
Expression();
jj_consume_token(SEMI);
jjtree.closeNodeScope(jjtn000, true);
jjtc000 = false;
{if (true) return jjtn000;}
} catch (Throwable jjte000) {
if (jjtc000) {
jjtree.clearNodeScope(jjtn000);
jjtc000 = false;
} else {
jjtree.popNode();
}
if (jjte000 instanceof RuntimeException) {
{if (true) throw (RuntimeException)jjte000;}
}
if (jjte000 instanceof ParseException) {
{if (true) throw (ParseException)jjte000;}
}
{if (true) throw (Error)jjte000;}
} finally {
if (jjtc000) {
jjtree.closeNodeScope(jjtn000, true);
}
}
throw new Error("Missing return statement in function");
}
final public void Expression() throws ParseException {
/*@bgen(jjtree) Expression */
SimpleNode jjtn000 = new SimpleNode(JJTEXPRESSION);
boolean jjtc000 = true;
jjtree.openNodeScope(jjtn000);
try {
AdditiveExpression();
} catch (Throwable jjte000) {
if (jjtc000) {
jjtree.clearNodeScope(jjtn000);
jjtc000 = false;
} else {
jjtree.popNode();
}
if (jjte000 instanceof RuntimeException) {
{if (true) throw (RuntimeException)jjte000;}
}
if (jjte000 instanceof ParseException) {
{if (true) throw (ParseException)jjte000;}
}
{if (true) throw (Error)jjte000;}
} finally {
if (jjtc000) {
jjtree.closeNodeScope(jjtn000, true);
}
}
}
final public void AdditiveExpression() throws ParseException {
/*@bgen(jjtree) AdditiveExpression */
SimpleNode jjtn000 = new SimpleNode(JJTADDITIVEEXPRESSION);
boolean jjtc000 = true;
jjtree.openNodeScope(jjtn000);
try {
MultiplicativeExpression();
label_1:
while (true) {
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case ADD:
case 27:
;
break;
default:
jj_la1[0] = jj_gen;
break label_1;
}
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case ADD:
jj_consume_token(ADD);
break;
case 27:
jj_consume_token(27);
break;
default:
jj_la1[1] = jj_gen;
jj_consume_token(-1);
throw new ParseException();
}
MultiplicativeExpression();
}
} catch (Throwable jjte000) {
if (jjtc000) {
jjtree.clearNodeScope(jjtn000);
jjtc000 = false;
} else {
jjtree.popNode();
}
if (jjte000 instanceof RuntimeException) {
{if (true) throw (RuntimeException)jjte000;}
}
if (jjte000 instanceof ParseException) {
{if (true) throw (ParseException)jjte000;}
}
{if (true) throw (Error)jjte000;}
} finally {
if (jjtc000) {
jjtree.closeNodeScope(jjtn000, true);
}
}
}
final public void MultiplicativeExpression() throws ParseException {
/*@bgen(jjtree) MultiplicativeExpression */
SimpleNode jjtn000 = new SimpleNode(JJTMULTIPLICATIVEEXPRESSION);
boolean jjtc000 = true;
jjtree.openNodeScope(jjtn000);
try {
UnaryExpression();
label_2:
while (true) {
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case MUL:
case 28:
case 29:
;
break;
default:
jj_la1[2] = jj_gen;
break label_2;
}
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case MUL:
jj_consume_token(MUL);
break;
case 28:
jj_consume_token(28);
break;
case 29:
jj_consume_token(29);
break;
default:
jj_la1[3] = jj_gen;
jj_consume_token(-1);
throw new ParseException();
}
UnaryExpression();
}
} catch (Throwable jjte000) {
if (jjtc000) {
jjtree.clearNodeScope(jjtn000);
jjtc000 = false;
} else {
jjtree.popNode();
}
if (jjte000 instanceof RuntimeException) {
{if (true) throw (RuntimeException)jjte000;}
}
if (jjte000 instanceof ParseException) {
{if (true) throw (ParseException)jjte000;}
}
{if (true) throw (Error)jjte000;}
} finally {
if (jjtc000) {
jjtree.closeNodeScope(jjtn000, true);
}
}
}
final public void UnaryExpression() throws ParseException {
/*@bgen(jjtree) UnaryExpression */
SimpleNode jjtn000 = new SimpleNode(JJTUNARYEXPRESSION);
boolean jjtc000 = true;
jjtree.openNodeScope(jjtn000);
try {
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case LPAREN:
jj_consume_token(LPAREN);
Expression();
jj_consume_token(RPAREN);
break;
case IDENTIFIER:
Identifier();
break;
case INTEGER_LITERAL:
Integer();
break;
default:
jj_la1[4] = jj_gen;
jj_consume_token(-1);
throw new ParseException();
}
} catch (Throwable jjte000) {
if (jjtc000) {
jjtree.clearNodeScope(jjtn000);
jjtc000 = false;
} else {
jjtree.popNode();
}
if (jjte000 instanceof RuntimeException) {
{if (true) throw (RuntimeException)jjte000;}
}
if (jjte000 instanceof ParseException) {
{if (true) throw (ParseException)jjte000;}
}
{if (true) throw (Error)jjte000;}
} finally {
if (jjtc000) {
jjtree.closeNodeScope(jjtn000, true);
}
}
}
final public void Identifier() throws ParseException {
/*@bgen(jjtree) Identifier */
SimpleNode jjtn000 = new SimpleNode(JJTIDENTIFIER);
boolean jjtc000 = true;
jjtree.openNodeScope(jjtn000);
try {
jj_consume_token(IDENTIFIER);
} finally {
if (jjtc000) {
jjtree.closeNodeScope(jjtn000, true);
}
}
}
final public void Integer() throws ParseException {
/*@bgen(jjtree) Integer */
SimpleNode jjtn000 = new SimpleNode(JJTINTEGER);
boolean jjtc000 = true;
jjtree.openNodeScope(jjtn000);
try {
jj_consume_token(INTEGER_LITERAL);
} finally {
if (jjtc000) {
jjtree.closeNodeScope(jjtn000, true);
}
}
}
/** Generated Token Manager. */
public lexicalTokenManager token_source;
SimpleCharStream jj_input_stream;
/** Current token. */
public Token token;
/** Next token. */
public Token jj_nt;
private int jj_ntk;
private int jj_gen;
final private int[] jj_la1 = new int[5];
static private int[] jj_la1_0;
static {
jj_la1_init_0();
}
private static void jj_la1_init_0() {
jj_la1_0 = new int[] {0x8002000,0x8002000,0x30004000,0x30004000,0x1020080,};
}
/** Constructor with InputStream. */
public lexical(java.io.InputStream stream) {
this(stream, null);
}
/** Constructor with InputStream and supplied encoding */
public lexical(java.io.InputStream stream, String encoding) {
try { jj_input_stream = new SimpleCharStream(stream, encoding, 1, 1); } catch(java.io.UnsupportedEncodingException e) { throw new RuntimeException(e); }
token_source = new lexicalTokenManager(jj_input_stream);
token = new Token();
jj_ntk = -1;
jj_gen = 0;
for (int i = 0; i < 5; i++) jj_la1[i] = -1;
}
/** Reinitialise. */
public void ReInit(java.io.InputStream stream) {
ReInit(stream, null);
}
/** Reinitialise. */
public void ReInit(java.io.InputStream stream, String encoding) {
try { jj_input_stream.ReInit(stream, encoding, 1, 1); } catch(java.io.UnsupportedEncodingException e) { throw new RuntimeException(e); }
token_source.ReInit(jj_input_stream);
token = new Token();
jj_ntk = -1;
jjtree.reset();
jj_gen = 0;
for (int i = 0; i < 5; i++) jj_la1[i] = -1;
}
/** Constructor. */
public lexical(java.io.Reader stream) {
jj_input_stream = new SimpleCharStream(stream, 1, 1);
token_source = new lexicalTokenManager(jj_input_stream);
token = new Token();
jj_ntk = -1;
jj_gen = 0;
for (int i = 0; i < 5; i++) jj_la1[i] = -1;
}
/** Reinitialise. */
public void ReInit(java.io.Reader stream) {
jj_input_stream.ReInit(stream, 1, 1);
token_source.ReInit(jj_input_stream);
token = new Token();
jj_ntk = -1;
jjtree.reset();
jj_gen = 0;
for (int i = 0; i < 5; i++) jj_la1[i] = -1;
}
/** Constructor with generated Token Manager. */
public lexical(lexicalTokenManager tm) {
token_source = tm;
token = new Token();
jj_ntk = -1;
jj_gen = 0;
for (int i = 0; i < 5; i++) jj_la1[i] = -1;
}
/** Reinitialise. */
public void ReInit(lexicalTokenManager tm) {
token_source = tm;
token = new Token();
jj_ntk = -1;
jjtree.reset();
jj_gen = 0;
for (int i = 0; i < 5; i++) jj_la1[i] = -1;
}
private Token jj_consume_token(int kind) throws ParseException {
Token oldToken;
if ((oldToken = token).next != null) token = token.next;
else token = token.next = token_source.getNextToken();
jj_ntk = -1;
if (token.kind == kind) {
jj_gen++;
return token;
}
token = oldToken;
jj_kind = kind;
throw generateParseException();
}
/** Get the next Token. */
final public Token getNextToken() {
if (token.next != null) token = token.next;
else token = token.next = token_source.getNextToken();
jj_ntk = -1;
jj_gen++;
return token;
}
/** Get the specific Token. */
final public Token getToken(int index) {
Token t = token;
for (int i = 0; i < index; i++) {
if (t.next != null) t = t.next;
else t = t.next = token_source.getNextToken();
}
return t;
}
private int jj_ntk() {
if ((jj_nt=token.next) == null)
return (jj_ntk = (token.next=token_source.getNextToken()).kind);
else
return (jj_ntk = jj_nt.kind);
}
private java.util.List<int[]> jj_expentries = new java.util.ArrayList<int[]>();
private int[] jj_expentry;
private int jj_kind = -1;
/** Generate ParseException. */
public ParseException generateParseException() {
jj_expentries.clear();
boolean[] la1tokens = new boolean[30];
if (jj_kind >= 0) {
la1tokens[jj_kind] = true;
jj_kind = -1;
}
for (int i = 0; i < 5; i++) {
if (jj_la1[i] == jj_gen) {
for (int j = 0; j < 32; j++) {
if ((jj_la1_0[i] & (1<<j)) != 0) {
la1tokens[j] = true;
}
}
}
}
for (int i = 0; i < 30; i++) {
if (la1tokens[i]) {
jj_expentry = new int[1];
jj_expentry[0] = i;
jj_expentries.add(jj_expentry);
}
}
int[][] exptokseq = new int[jj_expentries.size()][];
for (int i = 0; i < jj_expentries.size(); i++) {
exptokseq[i] = jj_expentries.get(i);
}
return new ParseException(token, exptokseq, tokenImage);
}
/** Enable tracing. */
final public void enable_tracing() {
}
/** Disable tracing. */
final public void disable_tracing() {
}
}