在之前的博客中己经分析了测试框架使用,博客链接如下https://blog.csdn.net/quyixiao/article/details/114679555,其实框架的源码大部分来自于 beanshell,只是我对其中的一些代码进行了改造,让他支持 python语法。整个过程分成读取,解析,执行三个过程,在下面的博客中就对这三个过程进行解析。
读取
读取主要是读取脚本中每一个字符,多个字符之间构成 token,每个 token 都有一个类型kind,如字符串for,表示for 类型,而 while 表示while类型,xxx表示identifier,也就是一个常量,1.2表示number 类型,如何区分不同字符串表示不同类型呢?不过在解析之前,我们先来看一个字符读取类JavaCharStream.java,为了测试方便,拷贝该类,修改其默认bufsize=8 ,默认数组扩容大小为4。
工具类JavaCharStream源码解析
JavaCharStreamTest.java
public class JavaCharStreamTest { public static final boolean staticFlag = false; public static final int DEFAULT_BUFSIZE = 4; public static final int DEFAULT_MAX_NEXT_CHAR_IND = 8; //字符16进制值转换函数 static final int hexval(char c) throws java.io.IOException { switch (c) { case '0': return 0; case '1': return 1; case '2': return 2; case '3': return 3; case '4': return 4; case '5': return 5; case '6': return 6; case '7': return 7; case '8': return 8; case '9': return 9; case 'a': case 'A': return 10; case 'b': case 'B': return 11; case 'c': case 'C': return 12; case 'd': case 'D': return 13; case 'e': case 'E': return 14; case 'f': case 'F': return 15; } throw new java.io.IOException(); // Should never come here } public int bufpos = -1; //最后一次读取字符所在 buffer 数组中的位置 public int bufsize; //buffer的长度 public int available; //buffer的可用长度 public int tokenBegin; //每一次获取 token时,在 buffer数组的起始位置 protected int bufline[]; //buffer 数组中的每一个字符对应在脚本中的行 protected int bufcolumn[]; //buffer 数组中每一个字符对应的列 protected int column = 0; //列下标从0开始 protected int line = 1; //行下标从1开始 protected boolean prevCharIsCR = false; protected boolean prevCharIsLF = false; protected java.io.Reader inputStream; //每一次从inputStream读取 byte[]数组,比如每次从inputStream读取指定长度的字符, //供 ReadByte()函数消费,当所有的 byte[]消费完时,再次从inputStream读取 protected char[] nextCharBuf; protected char[] buffer; //外部每一次读取的字符缓存数组 protected int maxNextCharInd = 0;//每次从 inputStream 中读取的固定长度 //nextCharBuf索引计数器,每ReadByte()一次,nextCharInd值加1, //当maxNextCharInd == nextCharInd 时,inputStream需要再次 reader byte 到nextCharBuf中,直到文件读取结束 protected int nextCharInd = -1; protected int inBuf = 0; //每次backup回退的byte数 //此函数主要用于buffer数组扩容 protected void ExpandBuff(boolean wrapAround) { char[] newbuffer = new char[bufsize + DEFAULT_BUFSIZE]; int newbufline[] = new int[bufsize + DEFAULT_BUFSIZE]; int newbufcolumn[] = new int[bufsize + DEFAULT_BUFSIZE]; try { if (wrapAround) { System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin); System.arraycopy(buffer, 0, newbuffer, bufsize - tokenBegin, bufpos); buffer = newbuffer; System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin); System.arraycopy(bufline, 0, newbufline, bufsize - tokenBegin, bufpos); bufline = newbufline; System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin); System.arraycopy(bufcolumn, 0, newbufcolumn, bufsize - tokenBegin, bufpos); bufcolumn = newbufcolumn; bufpos += (bufsize - tokenBegin); } else { System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin); buffer = newbuffer; System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin); bufline = newbufline; System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin); bufcolumn = newbufcolumn; bufpos -= tokenBegin; } } catch (Throwable t) { throw new Error(t.getMessage()); } available = (bufsize += DEFAULT_BUFSIZE); tokenBegin = 0; } //当nextCharBuf中的 byte 读取完时,从新从inputStream读取 byte[] 到nextCharBuf中,直到 //文件读取结束 protected void FillBuff() throws java.io.IOException { int i; if (maxNextCharInd == DEFAULT_MAX_NEXT_CHAR_IND) maxNextCharInd = nextCharInd = 0; try { if ((i = inputStream.read(nextCharBuf, maxNextCharInd, DEFAULT_MAX_NEXT_CHAR_IND - maxNextCharInd)) == -1) { inputStream.close(); throw new java.io.IOException(); } else maxNextCharInd += i; return; } catch (java.io.IOException e) { if (bufpos != 0) { --bufpos; backup(0); } else { bufline[bufpos] = line; bufcolumn[bufpos] = column; } throw e; } } //读取字符 protected char ReadByte() throws java.io.IOException { //当nextCharBuf数组中的byte[]读取完后,再从 inputStream 中读取 byte[] if (++nextCharInd >= maxNextCharInd) FillBuff(); return nextCharBuf[nextCharInd]; } //主要是提供给外部调用,当上一次调用了 backup函数时 //inBuf > 0 ,此时不再从nextCharBuf数组中读取,而是从缓存buffer中读取 public char BeginToken() throws java.io.IOException { if (inBuf > 0) { --inBuf; if (++bufpos == bufsize) bufpos = 0; tokenBegin = bufpos; return buffer[bufpos]; } tokenBegin = 0; bufpos = -1; return readChar(); } //当读取的字符超出available大小时,buffer数组扩容 protected void AdjustBuffSize() { if (available == bufsize) { if (tokenBegin > DEFAULT_BUFSIZE) { bufpos = 0; available = tokenBegin; } else { ExpandBuff(false); } } else if (available > tokenBegin) { available = bufsize; } else if ((tokenBegin - available) < DEFAULT_BUFSIZE) { ExpandBuff(true); } else { available = tokenBegin; } } ////每读取一个字符,都会追加字符所在的行和列 protected void UpdateLineColumn(char c) { column++; if (prevCharIsLF) { prevCharIsLF = false; line += (column = 1); } else if (prevCharIsCR) { prevCharIsCR = false; //如果读取到\n,行增加 if (c == '\n') { prevCharIsLF = true; } else line += (column = 1); } switch (c) { case '\r': prevCharIsCR = true; break; case '\n': prevCharIsLF = true; break; case '\t': //如果读取到\t ,列增加4个字节 column--; column += (8 - (column & 07)); break; default: break; } bufline[bufpos] = line; bufcolumn[bufpos] = column; } public char readChar() throws java.io.IOException { //如果上一次读取时有调用 backup 方法,此时inBuf大于0,直接读取buffer中的byte if (inBuf > 0) { --inBuf; if (++bufpos == bufsize) bufpos = 0; return buffer[bufpos]; } char c; if (++bufpos == available) AdjustBuffSize(); //下面这种情况主要是对\u2650,\u2650代表的字符是★ if ((buffer[bufpos] = c = ReadByte()) == '\\') { UpdateLineColumn(c); int backSlashCnt = 1; // 读取所有的 反斜杠 for (; ; ) { if (++bufpos == available) AdjustBuffSize(); try { if ((buffer[bufpos] = c = ReadByte()) != '\\') { UpdateLineColumn(c); if ((c == 'u') && ((backSlashCnt & 1) == 1)) { if (--bufpos < 0) bufpos = bufsize - 1; break; } backup(backSlashCnt); return '\\'; } } catch (java.io.IOException e) { if (backSlashCnt > 1) backup(backSlashCnt); return '\\'; } UpdateLineColumn(c); backSlashCnt++; } // 在这里,我们看到了奇数个反斜杠,后跟一个'u' try { while ((c = ReadByte()) == 'u') ++column; buffer[bufpos] = c = (char) (hexval(c) << 12 | hexval(ReadByte()) << 8 | hexval(ReadByte()) << 4 | hexval(ReadByte())); column += 4; } catch (java.io.IOException e) { throw new Error("Invalid escape character at line " + line + " column " + column + "."); } if (backSlashCnt == 1) return c; else { backup(backSlashCnt - 1); return '\\'; } } else { UpdateLineColumn(c); return (c); } } public int getColumn() { return bufcolumn[bufpos]; } public int getLine() { return bufline[bufpos]; } public int getEndColumn() { return bufcolumn[bufpos]; } public int getEndLine() { return bufline[bufpos]; } public int getBeginColumn() { return bufcolumn[tokenBegin]; } public int getBeginLine() { return bufline[tokenBegin]; } public void backup(int amount) { inBuf += amount; if ((bufpos -= amount) < 0) bufpos += bufsize; } public JavaCharStreamTest(java.io.Reader dstream, int startline, int startcolumn, int buffersize) { inputStream = dstream; line = startline; column = startcolumn - 1; available = bufsize = buffersize; buffer = new char[buffersize]; bufline = new int[buffersize]; bufcolumn = new int[buffersize]; nextCharBuf = new char[DEFAULT_MAX_NEXT_CHAR_IND]; } public JavaCharStreamTest(java.io.Reader dstream, int startline, int startcolumn) { this(dstream, startline, startcolumn, DEFAULT_MAX_NEXT_CHAR_IND); } public String GetImage() { if (bufpos >= tokenBegin) { return new String(buffer, tokenBegin, bufpos - tokenBegin + 1); } else { return new String(buffer, tokenBegin, bufsize - tokenBegin) + new String(buffer, 0, bufpos + 1); } } public static String read(JavaCharStreamTest jj_input_stream, int p) { StringBuilder sb = new StringBuilder(); for (int i = 0; i < p; i++) { try { char a = jj_input_stream.readChar(); sb.append(a + ""); } catch (IOException e) { e.printStackTrace(); } } return sb.toString(); } }
下面主要是几种测试用例。
public static void test1() throws Exception { Reader in = new FileReader("/Users/quyixiao/project/testshell/src/test/resources/test.tsh"); JavaCharStreamTest jj_input_stream = new JavaCharStreamTest(in, 1, 1); String a = read(jj_input_stream, 5); jj_input_stream.backup(7); String b = jj_input_stream.GetImage(); System.out.println(a); System.out.println(b); jj_input_stream.BeginToken(); String c = read(jj_input_stream, 17); System.out.println(c); }
public static void main(String[] args) throws Exception { //test2(); test3(); char c = '\u2605'; System.out.println(c); c = (char) (hexval('2') << 12 | hexval('6') << 8 | hexval('0') << 4 | hexval('5')); System.out.println(c); } public static void test2() throws Exception { Reader in = new FileReader("/Users/quyixiao/project/testshell/src/test/resources/test.tsh"); JavaCharStreamTest jj_input_stream = new JavaCharStreamTest(in, 1, 1); String a = read(jj_input_stream, 5); jj_input_stream.backup(7); String b = jj_input_stream.GetImage(); jj_input_stream.BeginToken(); String c = read(jj_input_stream, 13); System.out.println(c); jj_input_stream.BeginToken(); read(jj_input_stream, 10); System.out.println(jj_input_stream.GetImage()); } public static void test3() { for (int i = 0; i < 30; i++) { System.out.println(i + " " + (i & 1)); } }
关于为什么选择这几种测试,用户自己打断点调试一下吧,上面这个类如果不是很理解也没有关系,只要知道其使用方式即可,每次开始读时调用BeginToken方法,初始化读取的位置,每次调用readChar方法会返回一个字符,当本次读取完成时,调用GetImage()方法即可得到从调用BeginToken()方法后,所有读取到的字符,当读取到的字符不是想要的,可以调用 backup()方法,回退本次读取,如本次读取了 abcdef ,调用 backup(2), 再调用GetImage()方法,返回 abcd字符串,但下次再读取时,调用BeginToken()方法,再2个字符,返回的是 ef,调用 backup 方法主要是将本次己经读取过的字符,留给下次读取,可能上面理解很抽象,举2个例子吧。在文件中我们定义了一个字段 abcdefghik。
第一种情况:第一次读取,先调用BeginToken()方法,再调用3次readChar()方法,分别返回 a,b,c ,再调用GetImage(),此时返回字符串 abc,第二次读取,先调用BeginToken()方法,再调用3次readChar()方法,分别返回 d,e,f ,此次读取结束,调用GetImage()方法,返回 def。记得每次读取之前都需要调用BeginToken()方法。
第二种情况:先调用BeginToken()方法,再调用3次readChar()方法,分别返回 a,b,c,但是此时发现 bc 不是我想要的,只需要a即可,即可调用 backup(2),再调用 GetImage(),此时返回 a,第二次读取,先调用BeginToken()方法,再调用3次 readChar()方法,分别返回 b,c,d ,此时调用 GetImage()方法,返回字符串 bcd。虽然JavaCharStream类写得确实很完美,但是理解起来还是有一定难度的。
字符解析管理类 ParserTokenManager
从类名就可以得知,ParserTokenManager的主要功能就是解析字符封装成 token并返回,在解析ParserTokenManager这个类之前,我们先来看一个例子。
在上述测试过程中,最终for 循环表达式被解析成了for,(,i,in,range,(,5,),),{,print,(,a,),那么解析器是如何将单个字符组装成特定意义的类型的呢? 也就是说,读取三个字符 for,那么怎么知道 for就是我们想要的类型,而不是读取两个字符 fo 就返回identifier类型,也没有读取4个字符for(返回成identifier类型呢?带着疑问,我们来看源码 。
for(i in range(5)){ print(a) } 0 for : for 1 ( : ( 2 i : identifier 3 in : in 4 range : identifier 5 ( : ( 6 5 : number 7 ) : ) 8 ) : ) 9 { : { 10 : \n 11 print : identifier 12 ( : ( 13 a : identifier 14 ) : ) 15 : \n 16 } : } 17 : \n 18 : eof
ParserTokenManager.java
public class ParserTokenManager extends Utils implements ParserConstants { public static List<char[]> type = new ArrayList<>(); static { ... mapChar.put((int) '}', RBRACE); mapChar.put((int) '~', TILDE); } protected JavaCharStream input_stream; private final int[] jjrounds = new int[74]; private final int[] jjstateSet = new int[148]; protected char curChar; int curLexState = 0; int defaultLexState = 0; int jjnewStateCnt; int jjround; int jjmatchedPos; String jjmatchedKind; private boolean continueReader = false; private String curKind = ""; public ParserTokenManager(JavaCharStream stream) { if (JavaCharStream.staticFlag) { throw new Error("ERROR: Cannot use a static CharStream class with a non-static lexical analyzer."); } input_stream = stream; } public java.io.PrintStream debugStream = System.out; public void setDebugStream(java.io.PrintStream ds) { debugStream = ds; } public void ReInit(JavaCharStream stream) { jjmatchedPos = jjnewStateCnt = 0; curLexState = defaultLexState; input_stream = stream; ReInitRounds(); } private final void ReInitRounds() { int i; jjround = 0x80000001; for (i = 74; i-- > 0; ) jjrounds[i] = 0x80000000; } protected Token jjFillToken() { Token t = new Token(); t.kind = jjmatchedKind; t.image = input_stream.GetImage().trim(); if (jjmatchedKind.equals(EOF) && TStringUtil.isNotBlank(t.image)) { if (TRUE.equals(t.image.trim())) { t.kind = TRUE; } else if (FALSE.equals(t.image.trim())) { t.kind = FALSE; } } t.beginLine = input_stream.getBeginLine(); t.beginColumn = input_stream.getBeginColumn(); t.endLine = input_stream.getEndLine(); t.endColumn = input_stream.getEndColumn(); return t; } public static Token jjTempFillToken() { Token t = new Token(); t.kind = default_1; t.image = ""; t.beginLine = 0; t.beginColumn =0; t.endLine = 0; t.endColumn = 0; return t; } protected Token jjFillTokenEof() { Token t = new Token(); t.kind = jjmatchedKind; t.image = ""; return t; } public Token getNextToken() { Token specialToken = null; Token matchedToken; int curPos = 0; EOFLoop: for (; ; ) { try { curChar = input_stream.BeginToken(); } catch (java.io.IOException e) { jjmatchedKind = EOF; return jjFillTokenEof(); } jjmatchedKind = default_jjmatchedKind; // 2147483647 jjmatchedPos = 0; curPos = jjMoveStringLiteralDfa0_0(); if (curPos == -1) { // 如读取到续行符 \ 时,继续向下读取 continue; } if (jjmatchedKind != default_jjmatchedKind) { //文件读取结束 if (eqOR(jjmatchedKind, EOF)) { return jjFillToken(); } //如果返回的位置小于当前读取到的位置,回退到当前返回位置 // 如 buffer = [a,b,c,d,e,f,g] // jjmatchedPos = 3 // curPos = 6,则需要回退 6 - 3 - 1 = 2 ,因此下次从 e 开始读取 if (jjmatchedPos + 1 < curPos) { input_stream.backup(curPos - jjmatchedPos - 1); } // false jjmatchedKind is : [1, 2, 3, 4, 5, 6, 7, 8, 9, 61, 62, 63, 65, 70, 71] if (!commonJjstrLiteralImages.contains(jjmatchedKind)) { matchedToken = jjFillToken(); matchedToken.specialToken = specialToken; return matchedToken; } else { // false jjmatchedKind is : if (!jjstrLiteralImages.contains(jjmatchedKind)) { matchedToken = jjFillToken(); if (specialToken == null) specialToken = matchedToken; else { matchedToken.specialToken = specialToken; specialToken = (specialToken.next = matchedToken); } } continue EOFLoop; } } //如果读取不正确,返回出错的行和列 int error_line = input_stream.getEndLine(); int error_column = input_stream.getEndColumn(); String error_after = null; boolean EOFSeen = false; try { input_stream.readChar(); input_stream.backup(1); } catch (java.io.IOException e1) { EOFSeen = true; error_after = curPos <= 1 ? "" : input_stream.GetImage(); if (curChar == '\n' || curChar == '\r') { error_line++; error_column = 0; } else error_column++; } if (!EOFSeen) { input_stream.backup(1); error_after = curPos <= 1 ? "" : input_stream.GetImage(); } throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR); } } private final int jjStartNfaWithStates_0(String kind) { //continueReader 主要是用于处理字符串,只要""或''包裹的任意字符,都当成字符串来处理 if (!continueReader && eqOR(kind, TAB, SPACE)) { //如果遇到 tab ,空格, 和逗号 return getCommon(); } else if (!continueReader && eqOR(kind, ENTER, NEXT_LINE)) { //当读取到\r和 \n String image = input_stream.GetImage().trim(); if (image.length() > 0) { input_stream.backup(1); } else { jjmatchedPos = input_stream.bufpos; jjmatchedKind = NEXT_LINE; return 1; } return getCommon(); } else if (!continueReader && eqOR(kind, CONTINUE_LINE)) { //当读取到续行符 \ 时 char c = readChar(); if (c > 0) { input_stream.backup(1); } String temp = c + ""; if (c == 0 || TStringUtil.isBlank(temp)) { //如果续行符前面的类型不为空时,返回续行符前面的内容 String image = input_stream.GetImage().trim(); if (image.length() > 1) { input_stream.backup(1); return getCommon(); } else { TTuple2<Boolean, Integer> line = readUtilChar('\n').getData(); int i = line.getSecond() + input_stream.GetImage().length() - 1; input_stream.tokenBegin += i; return -1; } } } else if (!continueReader && eqOR(kind, LPAREN, RPAREN, LBRACE, RBRACE, LBRACKET, RBRACKET, SEMICOLON, COMMA, COLON, HOOK,AT,TILDE)) { //如果读取到 ( ){ } [ ] ; , : ? @ ~ String image = input_stream.GetImage().trim(); if (eqOR(image, SEMICOLON)) { //如果是; ,略过, 在 java中代码必需以;结尾,但是在脚本中,写不写;都不影响 input_stream.tokenBegin += 1; return -1; } if (image.length() > 1) { input_stream.backup(1); } return getCommon(); } else if (!continueReader && eqOR(kind, DOT)) { //. 的处理 String image = input_stream.GetImage().trim(); image = image.substring(0, image.length() - 1); if (TStringUtil.isNumber(image)) { //如果前面是数字,则继续读取 return 0; } if (image.length() > 0) { input_stream.backup(1); } return getCommon(); } else if (!continueReader && eqOR(kind, LT)) { //如果是 < ,可能是<=,<<=,<< 三种情况 return getSpecial(new char[]{'<'}, '<', '='); } else if (!continueReader && eqOR(kind, GT)) {//可能会出现的情况>=,>>,>>>,>>=,>>>= return getSpecial(new char[]{'>'}, '=', '>'); } else if (!continueReader && eqOR(kind, SLASH)) { // / 号后面只能接 = 但是,如果 //,/**... */ 表示注释,下面主要是对注释处理 char c = readChar(); input_stream.backup(1); if (c == '/') { //表示注释 TTuple2<Boolean, Integer> line = readUtilChar('\n').getData(); if (line.getFirst()) { input_stream.backup(1); } int i = line.getSecond() + input_stream.GetImage().length() - 1; input_stream.tokenBegin += i; return -1; //返回-1 表示略过 所解析的字符 , //表示注释 } else if (c == '*') { // /* .... */ 也表示注释 readChar(); //读取当前 * int i = input_stream.GetImage().length() - 1; while (true) { TTuple2<Boolean, Integer> line = readUtilChar('*').getData(); i += line.getSecond(); char temp = readChar(); if (temp == '/') { i = i + 1; break; } } input_stream.tokenBegin += i; return -1; } return getSpecial(null, '='); } else if (!continueReader && eqOR(kind, ASSIGN)) { // = ,后面只能接 =,如 ==,<=,<<=,>=,>>=,>>>=,!=,+=,-=,*=,/=,&=,|=,^=,%= return getSpecial(new char[]{'=', '<', '>', '!', '+', '-', '*', '/', '&', '|', '^', '%'}, '='); } else if (!continueReader && eqOR(kind, STAR)) { // ! ,* ,^,% ,后面只能接 = return getSpecial(new char[]{'*'}, '*','='); } else if (!continueReader && eqOR(kind, BANG, XOR, MOD)) { // ! ,* ,^,% ,后面只能接 = return getSpecial(null, '='); } else if (!continueReader && eqOR(kind, PLUS)) { // + 号后面能接 +,= 有 ,++ 和+= 两种情况 return getSpecial(new char[]{'+'}, '+', '='); } else if (!continueReader && eqOR(kind, MINUS)) { // - 号后面 只能接 - 或 = ,组成 -= 或-- return getSpecial(new char[]{'-'}, '-', '='); } else if (!continueReader && eqOR(kind, AND)) { // & 后面只能接 & 或 = return getSpecial(new char[]{'&'}, '&', '='); } else if (!continueReader && eqOR(kind, OR)) { // | 后面只能接 | 或 = return getSpecial(new char[]{'|'}, '|', '='); } else if (eqOR(kind, DOUBLE_QUOT, SINGLE_QUOT)) { // " 双引号处理,' 单引号处理 if (continueReader && eqOR(kind, curKind)) { input_stream.backup(2); char c = readChar(); readChar(); if (c == '\\') {//表示继续 return 0; } continueReader = !continueReader; curKind = ""; int position = getCommon(); jjmatchedKind = STR; return position; } if (!continueReader) { continueReader = !continueReader; curKind = kind; } } else if (eqOR(kind, EOF)) { jjmatchedKind = EOF; return 1; } return 0; } public char readChar() { try { return input_stream.readChar(); } catch (IOException e) { } return 0; } public int getCommon() { String image = input_stream.GetImage().trim(); if (TStringUtil.isNotBlank(image)) { //表明前面有非空格,tab,的字符 // 如 image = for ,和我们的关键字 for 匹配上了,则 match = for String match = matchs(image.trim().toCharArray()); jjmatchedPos = input_stream.bufpos; if (TStringUtil.isNotBlank(match)) { jjmatchedKind = match; //设置 匹配到的类型 } else { if (TStringUtil.isNumber(image)) { //判断当前字符串是不是数字类型 jjmatchedKind = NUMBER; } else { jjmatchedKind = IDENTIFIER; //如果既不是关键字类型,也还是数字类型,则只能是identifier类型了 } } return 2; } return 0; } ... public int jjMoveStringLiteralDfa0_0() { while (true) { try { int flag = 0; if (curChar == '\t') { //如果是 TAB 类型 flag = jjStartNfaWithStates_0(TAB); } else if (curChar == ' ') { //如果是 空格 类型 flag = jjStartNfaWithStates_0(SPACE); } else if (curChar == '\r') { //如果是 回车 类型 flag = jjStartNfaWithStates_0(ENTER); } else if (curChar == '\n') { //如果是换行符 flag = jjStartNfaWithStates_0(NEXT_LINE); } else if ((curChar >= 48 && curChar <= 57) || (curChar >= 65 && curChar <= 90) || (curChar >= 97 && curChar <= 122)) { //如果是 a~z,A~Z,0~9 三种情况,表示是字面量类型 flag = jjStartNfaWithStates_0(IDENTIFIER); } else {//@,[,\,],^,_,`,!,",#,$,%,&,',(,),*,+,-,.,/,:,;,{,<,|,=,},>,~,? 类型 flag = jjStartNfaWithStates_0(mapChar.get(new Integer(curChar))); } if (flag > 0) { //如果 flag > 0 ,表示本次读取结束 ,如读取到 for 了 return input_stream.bufpos + 1; } if (flag == -1) { // 当读取到续行符 \ 时,表示不要返回,继续读取 return -1; } curChar = input_stream.readChar(); //继续向后读取一个字符 } catch (IOException e) { jjStartNfaWithStates_0(SPACE); return input_stream.bufpos + 1; } catch (Exception e) { e.printStackTrace(); return input_stream.bufpos + 1; } } } }
上述代码中需要注意方法是getSpecial方法,这个方法中传递了两个参数,第一个参数是当前字符之前能匹配的字符数组,而第二个参数是当前字符之后能匹配字符数组,什么意思呢?如解析到当前字符是=,等于号出现的所有可能情况是 ==,<=,<<=,>=,>>=,>>>=,!=,+=,-=,=,/=,&=,|=,^=,%=,因此只要等于号前面的字符是’=’, ‘<’, ‘>’, ‘!’, ‘+’, ‘-’, '’, ‘/’, ‘&’, ‘|’, ‘^’, ‘%’ 及后面的字符是’='的情况,都不终止本次读取,+号前面允许是+号,后面允许是=号,可能比较抽象,来举个例子,
a+= 3 ,先读取 a,再读取+号,此时发现+号前面不是+号,因此终止本次读取,解析得到字符串 a ,再向后读取到=号,发现=号前面是+号,但是=后面不是=号,因此终止本次读取,得到 +=符号,+=号就属于 +=类型,继续向后读取3 ,发现3是一个 number 类型,所以 a += 3 己经被解析完成。
经过上面的jjStartNfaWithStates_0方法的分析,我们理解了字符解析器的工作流程,如果遇到空格,TAB 必然终止本次读取,获取读取的 token返回,getCommon()方法还是需要注意一下的,在 getCommon()方法中调用input_stream.GetImage()得到本次解析的字符串,然后将字符串与
【break , case , catch , continue , default , do , else , false , finally , for , if , null , return , switch , throw , throws , true , try , while , def , export , ( , ) , { , } , [ , ] , ; , : , , , . , in , ? , ** , ~ , @ , new , import , map , lambda , global , = , > , < , ! , | , & , + , - , * , / , ^ , % , == , <= , >= , != , ++ , – , && , || , << , >> , >>> , += , -= , *= , /= , &= , |= , ^= , %= , <<= , >>= , >>>= 】进行匹配,如果匹配到了,如果当前解析到的字符串是 return,那么其类型【kind】就是 return 类型,如果当前解析到的字符串是 import,其类型就是 import 类型,如果不是上述类型,再判断当前字符串是不是 number 类型,如果不是 number 类型,只可能是identifier类型。
通过上述分析,我们知道了字符解析器解析流程,而解析器得到的数据最终被封装成 Token,下面我们来看看Token的数据结构。
public class Token implements java.io.Serializable { //当前字符串的类型,如 for,number,identifier类型 public String kind; //当前字符串的开始行,开始列,结束行,结束列 public int beginLine, beginColumn, endLine, endColumn; //当前字符串的内容 public String image; //当前 token 的下一个 token, 如 a = 1 , a 的 token 的 next token 是 = 号所代表的 token public Token next; public Token specialToken; public String toString() { return image; } //创建新的 token ,和 new Token()一样 public static final Token newToken(int ofKind) { switch (ofKind) { default: return new Token(); } } }
ParserTokenManager类保证了获取正确的 token,下面就来看看执行树的创建过程,执行树主要是在 Parser 类中创建。
执行树的创建
在理解执行树的创建过程前,我们来解析一下整个源码解析器从解析到执行的整个过程。
上图中对 a = 1 + 2 + 3 执行做了简单的分析,像这种解析执行方法在 Spring 的 SpEL表达式的源码中也使用到,那么我们还是来分析 Parser 类是如何创建执行树的。
Parser.java
final public boolean BlockStatement() throws ParseException { //消费掉所有的的行符 jj_consume_token_next_line(); if (jj_2_33(3, EOF)) return false; //如果文件己经结束,则终止执行树的创建 if (isMethod()) { //如果是方法声明,我们定义以 def 开头的都是方法声明 MethodDeclaration(); } else if (jj_2_31(2147483647)) { //变量声明,只要是 变量=的情况都是变量声明,也可以是 变量,变量...变量 = 的情况 VariableDeclarator(); } else if (jj_2_33(3, AT)) { //当解析到@时,表明是一个注解 AnnotationMethodInvocationDeclarator(); } else if (jj_2_33(3, IMPORT)) { //当解析到import时,表示是引入一个变量或源码 ImportDeclaration(); } else if (jj_2_33(3, GLOBAL)) { // global 定义全局变量 GlobalDeclaration(); } else { Statement(); //普通表达式的创建 } return true; } private void MethodDeclaration() throws ParseException { //如果是方法的声明,创建TSHMethodDeclaration节点 TSHMethodDeclaration jjtn000 = new TSHMethodDeclaration(T_MethodDeclaration); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); Token t = null; try { //消费掉 def jj_consume_token(DEF); //消费掉方法名 t = jj_consume_token(IDENTIFIER); jjtn000.methodName = t.image; //设置方法名 FormalParameters(); switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case LBRACE: Block(); break; default: jj_la1[8] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } // 块内解析 final public void Block() throws ParseException { TSHBlock jjtn000 = new TSHBlock(T_Block); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); Token s = null; try { jj_consume_token(LBRACE); //消费掉 { label_22: while (true) { // jj_2_23()方法的判断很简单,如果遇到 } ,则终止循环,可能有人会犯诋估了, 如果 @1{ ... @2{ ... @3} ... @4} //这种情况,上面消费掉@1的{ ,如果消费掉@3 } 怎么办呢? // 大家不用担心,因为@1所对应的 { ,只可能消费掉@4 对应的 } // 因为在解析@2时 { ,会成对的将@3 } 消费掉,因此,@1所对应的{,只能是@4对应的 },因此在消费掉一个{后,只需要等到 // 下一个}出现,即可退出 {} 内节点的解析 if (jj_2_23(1)) { ; } else { break label_22; } //继续块的解析 BlockStatement(); } jj_consume_token_util(RBRACE); // } ,为什么会直到消费掉 }呢?因为在 }之前可能会有多个回车换行 jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); } catch (Throwable jjte000) { if (jjtc000) { jjtree.clearNodeScope(jjtn000); jjtc000 = false; } else { jjtree.popNode(); } if (jjte000 instanceof RuntimeException) { throw (RuntimeException) jjte000; } if (jjte000 instanceof ParseException) { throw (ParseException) jjte000; } { if (true) throw (Error) jjte000; } } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } final public void LambdaBlock() throws ParseException { TSHLambdaBlock jjtn000 = new TSHLambdaBlock(T_LambdaBlock); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { BlockStatement(); jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } //方法参数出现的形式 (a,b,c) 或 (a = 1 ,b = 2 ,c ) 或 (a = 1 ,* args ,**kwargs )或(a = 1 , b = lambda x ,y : x + y )等 final public void FormalParameters() throws ParseException { //方法参数列表对象 TSHFormalParameters jjtn000 = new TSHFormalParameters(T_FormalParameters); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { jj_consume_token(LPAREN); //消费方法名后的( switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case STAR: case SSTAR: case IDENTIFIER: //如果当前参数是*,**,或 字符常量,创建子节点FormalParameter FormalParameter(); label_3: while (true) { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case COMMA: ; break; default: jj_la1[16] = jj_gen; break label_3; } jj_consume_token(COMMA); //消费掉逗号 FormalParameter(); //再次创建子节点 } break; default: jj_la1[17] = jj_gen; } jj_consume_token(RPAREN); } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } final public void FormalParameter() throws ParseException { TSHFormalParameter jjtn000 = new TSHFormalParameter(T_FormalParameter); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); Token t; try { if (jj_2_33(3, STAR)) { // def fn(x, y,*args) 一个* 的情况 jj_consume_token(STAR); jjtn000.kind = STAR; } else if (jj_2_33(3, SSTAR)) { // def fn(x, y,**kwargs) 2 个 * 的情况 jj_consume_token(SSTAR); jjtn000.kind = SSTAR; } t = jj_consume_token(IDENTIFIER); //消费得到方法参数变量名 jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); jjtn000.name = t.image; if (jj_2_33(3, ASSIGN)) { //如果没有扫描到=号,则变量的定义肯定是(a ,b) jj_consume_token(ASSIGN); //如果扫描到=号,则解析等号后面的表达式,如(a = lambda x ,y : x + y ) Expression(); } } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } final public void LambdaFormalParameters() throws ParseException { TSHFormalParameters jjtn000 = new TSHFormalParameters(T_FormalParameters); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case STAR: case SSTAR: case IDENTIFIER: FormalParameter(); label_3: while (true) { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case COMMA: ; break; default: jj_la1[16] = jj_gen; break label_3; } jj_consume_token(COMMA); FormalParameter(); } break; default: jj_la1[17] = jj_gen; } } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } final public void Statement() throws ParseException { jj_consume_token_next_line(); //消费掉所有的行 if (jj_3_40(3)) { //如果是 三目运算符 如 a > b ? a : b LabeledStatement(); // break lable_;的情况 } else { doStatement(); // 正在开始解析表达式 } } final public void doStatement() throws ParseException { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case LBRACE: //如果是 { ,表明是一个执行块 ,如 { c= a + b \n print(c ,a ,b )}等 Block(); break; case FALSE: case NULL: case TRUE: case STR: case IDENTIFIER: case LPAREN: case LBRACKET: case BANG: case TILDE: case INCR: case DECR: case PLUS: case MINUS: case MAP: StatementExpression(); // 如 map1 = {'username':'zhangsan' ,'age':18} ... break; case SWITCH: SwitchStatement(); // swith(...) { case condition1: ... break \n case condition2 : ... break label_1 \n default : ... break } break; case IF: IfStatement(); //if(condition) {...} break; case WHILE: WhileStatement(); //while(condition) { ...} break; case DO: DoStatement(); // do {...} while(condition) break; default: jj_la1[69] = jj_gen; switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case FOR: ForAfterStatement(); // for (i in range( 5 )) { ...} break; case BREAK: BreakStatement(); // break 或break label_1 break; case CONTINUE: // continue 或continue label_1 ContinueStatement(); break; case RETURN: // return a 或return a ,b ,c ReturnStatement(); break; case THROW: ThrowStatement(); // throw new Exception(...) break; case TRY: TryStatement(); //try {...} catch (..) { ...} break; case EXPORT: // export a ,b ,c 或 export a ,a + b ExportStatement(); break; case EOF: break; default: jj_la1[70] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } } } //while 表达式解析 final public void WhileStatement() throws ParseException { TSHWhileStatement jjtn000 = new TSHWhileStatement(T_WhileStatement); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { jj_consume_token(WHILE); jj_consume_token(LPAREN); Expression(); jj_consume_token(RPAREN); Statement(); } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } // do { ...} while( condition ) ,do while 表达式解析 final public void DoStatement() throws ParseException { TSHWhileStatement jjtn000 = new TSHWhileStatement(T_WhileStatement); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { jj_consume_token(DO); Statement(); jj_consume_token(WHILE); jj_consume_token(LPAREN); Expression(); jj_consume_token(RPAREN); jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); jjtn000.isDoStatement = true; } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } // 三目运算符解析 a > b ? a > c ? a : c : b > c ? b : c ,计算 a b c 的最大值 final public void LabeledStatement() throws ParseException { TSHLabeledStatement jjtn000 = new TSHLabeledStatement(T_LabeledStatement); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { Token t = jj_consume_token(IDENTIFIER); jj_consume_token(COLON); //消费掉 : jj_consume_token_next_line(); // 可能会出现 a > b ? sum(a - b ) : sub(b - a ) ,计算绝对值,因此Statement()解析的不一定是一个变量名, //也可能是一个表达式,因此调用Statement Statement(); jjtn000.label = t.image; } catch (Throwable jjte000) { .... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } final public void StatementExpression() throws ParseException { //表达式解析 Expression(); } // swith ( ... ) { case condition1 : ....; condition2 : ... break label_1 } 的情况 final public void SwitchStatement() throws ParseException { TSHSwitchStatement jjtn000 = new TSHSwitchStatement(T_SwitchStatement); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { jj_consume_token(SWITCH); jj_consume_token(LPAREN); Expression(); jj_consume_token(RPAREN); jj_consume_token(LBRACE); label_23: while (true) { jj_consume_token_next_line(); switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case CASE: case _DEFAULT: ; break; default: jj_la1[73] = jj_gen; break label_23; } SwitchLabel(); int i = 0; label_24: while (true && i != 1) { // case condition1 : condition2 : .... break 的情况 i = jj_2_29(1); if (i == 2 || i == 1) { //如果是 i = 1 ,则下次退出 ; } else { //如果遇到 case 或 } 情况,直接终止到label_24 break label_24; } BlockStatement(); } } jj_consume_token_util(RBRACE); } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } // switch 表达式解析,switch (...) { case condition: ... ; default : ... ; } final public void SwitchLabel() throws ParseException { TSHSwitchLabel jjtn000 = new TSHSwitchLabel(T_SwitchLabel); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case CASE: jj_consume_token(CASE); Expression(); jj_consume_token(COLON); break; case _DEFAULT: jj_consume_token(_DEFAULT); jj_consume_token(COLON); jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); jjtn000.isDefault = true; break; default: jj_la1[74] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } // if 表达式 if( condition ) { ... } final public void IfStatement() throws ParseException { TSHIfStatement jjtn000 = new TSHIfStatement(T_IfStatement); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { jj_consume_token(IF); jj_consume_token(LPAREN); Expression(); jj_consume_token(RPAREN); Statement(); switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case ELSE: //如果有 else ,如 if(condition) { ... }else {...} 的情况 jj_consume_token(ELSE); Statement(); break; default: jj_la1[75] = jj_gen; ; } } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } // for 表达式在之前的情况,为[]集体赋值 ,如 [x + 1 for (x in range(10))] 得到 [1,2,3,4,5,6,7,8,9,10] final public void ForBeforeStatement() throws ParseException { TSHBeforeForStatement jjtn000 = new TSHBeforeForStatement(T_BeforeForStatement); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); Token t = null; try { doStatement(); jj_consume_token(FOR); // for jj_consume_token(LPAREN); // ( t = jj_consume_token(IDENTIFIER); // i Token t1 = jj_consume_token_next(); if (eq(t1.kind, COMMA)) { // for(i,item in range(1,5)){} 的情况 jjtn000.kOrI = t.image; Token t2 = jj_consume_token(IDENTIFIER); // i jjtn000.varName = t2.image; jj_consume_token(IN); // in } else { //如果没有消费到, 的话,肯定消费了 in jjtn000.varName = t.image; } Expression(); // range(1,10) jj_consume_token(RPAREN); // ) jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } // for(i in range(5)) { ... } ,for表达式解析 final public void ForAfterStatement() throws ParseException { TSHForStatement jjtn000 = new TSHForStatement(T_ForStatement); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); Token t = null; try { jj_consume_token(FOR); // for jj_consume_token(LPAREN); // ( t = jj_consume_token(IDENTIFIER); // i Token t1 = jj_consume_token_next(); if (eq(t1.kind, COMMA)) { // for(i,item in range(1,5)){} 的情况 jjtn000.kOrI = t.image; Token t2 = jj_consume_token(IDENTIFIER); // i jjtn000.varName = t2.image; jj_consume_token(IN); // in } else { //如果没有消费到, 的话,肯定消费了 in jjtn000.varName = t.image; } Expression(); // range(1,10) jj_consume_token(RPAREN); // ) Statement(); // {} jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } // break 或 break label_1 两种情况 final public void BreakStatement() throws ParseException { TSHReturnStatement jjtn000 = new TSHReturnStatement(T_ReturnStatement); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { jj_consume_token(BREAK); switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case IDENTIFIER: // break label_1 的情况 Token t = jj_consume_token(IDENTIFIER); jjtn000.label = t.image; break; default: jj_la1[83] = jj_gen; ; } jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); jjtn000.kind = BREAK; } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } // continue 块 final public void ContinueStatement() throws ParseException { // continue 和 continue label_1; TSHReturnStatement jjtn000 = new TSHReturnStatement(T_ReturnStatement); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { jj_consume_token(CONTINUE); switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case IDENTIFIER: // continue label_1 的解析 Token t = jj_consume_token(IDENTIFIER); jjtn000.label = t.image; break; default: jj_la1[84] = jj_gen; ; } jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); jjtn000.kind = CONTINUE; } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } // return 表达式 final public void ReturnStatement() throws ParseException { TSHReturnStatement jjtn000 = new TSHReturnStatement(T_ReturnStatement); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { jj_consume_token(RETURN); // return sum(a + b ) Expression(); while (jj_2_33(3, COMMA)) { //下面是 return sum(a + b ) ,a ,b 返回一个元组 jj_consume_token(COMMA); Expression(); } jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); jjtn000.kind = RETURN; } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } // throw new Exception()表达式解析 final public void ThrowStatement() throws ParseException { TSHThrowStatement jjtn000 = new TSHThrowStatement(T_ThrowStatement); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { jj_consume_token(THROW); //消费掉 throw Expression(); // 会被AllocationExpression方法解析到,new Exception(...) } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } // try 表达式 ,java 中的语法 try { \n ... \n } catch(e){ \n e.printStackTrace() \n}finally{\n .... \n } final public void TryStatement() throws ParseException { TSHTryStatement jjtn000 = new TSHTryStatement(T_TryStatement); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); boolean closed = false; try { jj_consume_token(TRY); Block(); // {} 块表达式解析 label_27: while (true) { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case CATCH: ; break; default: jj_la1[86] = jj_gen; break label_27; } jj_consume_token(CATCH); jj_consume_token(LPAREN); FormalParameter(); // 解析 catch( e ) 的方法参数 jj_consume_token(RPAREN); Block(); // 再解析 catch 后的{}表达式 closed = true; } switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case FINALLY: jj_consume_token(FINALLY); Block(); closed = true; break; default: jj_la1[87] = jj_gen; } jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); if (!closed) { if (true) throw new ParseException(); } } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } // export 表达式解析 final public void ExportStatement() throws ParseException { TSHExportStatement jjtn000 = new TSHExportStatement(T_ExportStatement); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { //export sum(a,b) , (lambda x ,y : x - y )(5,4) 这种写法 //export a ,b 都可以,因此第一个参数是 export,每个表达式都是,隔开 jj_consume_token(EXPORT); Expression(); while (jj_2_33(3, COMMA)) { jj_consume_token(COMMA); Expression(); } jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); jjtn000.kind = RETURN; } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } //变量声明 final public void VariableDeclarator() throws ParseException { TSHVariableDeclarator jjtn000 = new TSHVariableDeclarator(T_VariableDeclarator); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); Token t; try { List<String> list = new ArrayList<>(); t = jj_consume_token(IDENTIFIER); //变量声明的第一个参数肯定是identifier list.add(t.image); if (!jj_2_33(3, ASSIGN)) { //如果变量后面第一个参数不是=号,申明了多个变量名 a ,b .... = [] do { jj_consume_token(COMMA); t = jj_consume_token(IDENTIFIER); list.add(t.image); } while (!jj_2_33(3, ASSIGN)); //直到出现等号结束变量 } switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case ASSIGN: jj_consume_token(ASSIGN); VariableInitializer(); //变量初始化 break; default: jj_la1[12] = jj_gen; } jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); jjtn000.names = list.toArray(new String[list.size()]); } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } //变量初始化 final public void VariableInitializer() throws ParseException { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case LBRACKET: // 如果 [开头,表示的是list 初始化 ListInitializer(); break; case LBRACE: //如果{开头,表示map 初始化 MapInitializer(); break; case LPAREN: case FALSE: case NULL: case TRUE: case STR: case NUMBER: case IDENTIFIER: case BANG: case TILDE: case INCR: case DECR: case PLUS: case MINUS: if (jj_3_42(10)) { // 可能是 [x for ( i in range(5))] ,这种情况 ForBeforeStatement(); } else { Expression(); //普通表达式的解析 } break; default: jj_la1[13] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } } //下面这种情况,注解情况的处理 ,如@logger \n def method( a ){\n print(a) \n} 这种情况处理 final public void AnnotationMethodInvocationDeclarator() throws ParseException { TSHAnnotationMethodDeclaration jjtn000 = new TSHAnnotationMethodDeclaration(T_AnnotationMethodDeclaration); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { jj_consume_token(AT); MethodInvocation(); //方法调用解析 BlockStatement(); // 方法申明解析 } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } // import 开头的变量声明, import 有两种情况, import a ,b 这种还有 import common.tsh 引入公共方法 final public void ImportDeclaration() throws ParseException { TSHImportDeclaration jjtn000 = new TSHImportDeclaration(T_ImportDeclaration); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); Token t = null; StringBuilder sb = new StringBuilder(); try { jj_consume_token(IMPORT); // import 以,隔开 while (true) { if (jj_2_33(3, COMMA)) { jj_consume_token(COMMA); } t = jj_consume_token(IDENTIFIER); //消费变量名 StringBuffer s = new StringBuffer(t.image); label_5: while (true) { if (jj_2_7(2)) { ; } else { break label_5; } jj_consume_token(DOT); t = jj_consume_token(IDENTIFIER); s.append("." + t.image); } sb.append(s.toString()).append(","); if (jj_2_33(3, NEXT_LINE)) { break; } } jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); jjtn000.text = sb.toString(); } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } //变量全局变量 global a = 1 ,在所有的后续脚本调用中都可以修改,引用此变量 final public void GlobalDeclaration() throws ParseException { TSHGlobalStatement jjtn000 = new TSHGlobalStatement(T_GlobalDeclaration); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { jj_consume_token(GLOBAL); Expression(); } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } //表达式解析 final public void Expression() throws ParseException { //等于表达式 ,如 a,b,c.... = .... 的情况 //也可能是 a += b ,a -=b , a >>>=b ,a <<=b 等等 if (jj_2_8(2147483647)) { Assignment(); } else { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case BOOL: case FALSE: case NULL: case TRUE: case IDENTIFIER: case LPAREN: case BANG: case STR: case NUMBER: case TILDE: case INCR: case DECR: case NEW: case PLUS: case MINUS: ConditionalExpression(); break; case LBRACKET: ListInitializer(); break; case LBRACE: MapInitializer(); break; case STAR: case SSTAR: StarArgument(); // * ,** 表达式处理 break; case LAMBDA: LambdaDeclaration(); break; case MAP: MapInvocation(); break; default: jj_la1[23] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } } } // list 类型初始化 final public void ListInitializer() throws ParseException { TSHListInitializer jjtn000 = new TSHListInitializer(T_ListInitializer); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { jj_consume_token(LBRACKET); switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case NUMBER: case FALSE: case NULL: case TRUE: case STR: case IDENTIFIER: case LPAREN: case LBRACE: case BANG: case TILDE: case INCR: case DECR: case PLUS: case MINUS: VariableInitializer(); label_2: while (true) { if (jj_2_4(2)) { ; } else { break label_2; } jj_consume_token(COMMA); VariableInitializer(); } break; default: jj_la1[14] = jj_gen; } switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case COMMA: jj_consume_token(COMMA); break; default: jj_la1[15] = jj_gen; ; } jj_consume_token(RBRACKET); } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } /* { 'abc': 123, 98.6: 37 } 写法 */ final public void MapInitializer() throws ParseException { TSHMapInitializer jjtn000 = new TSHMapInitializer(T_MapInitializer); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { jj_consume_token(LBRACE); switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case NUMBER: case FALSE: case NULL: case TRUE: case STR: case IDENTIFIER: case LPAREN: case LBRACE: case BANG: case TILDE: case INCR: case DECR: case PLUS: case MINUS: VariableInitializer(); jj_consume_token(COLON); VariableInitializer(); label_2: while (true) { if (jj_2_4(2)) { ; } else { break label_2; } jj_consume_token(COMMA); VariableInitializer(); jj_consume_token(COLON); VariableInitializer(); } break; default: jj_la1[14] = jj_gen; } switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case COMMA: jj_consume_token(COMMA); break; default: jj_la1[15] = jj_gen; } jj_consume_token(RBRACE); } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } // * 类型参数 final public void StarArgument() throws ParseException { TSHStarArgument jjtn000 = new TSHStarArgument(T_StarArgument); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { Token t1 = jj_consume_token_next(); jjtn000.kind = t1.kind; Expression(); } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } // lambda表达式声明 private void LambdaDeclaration() throws ParseException { TSHLambdaDeclaration jjtn000 = new TSHLambdaDeclaration(T_LambdaDeclaration); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { jj_consume_token(LAMBDA); jjtn000.methodName = TOrderUtil.getMethodName(); LambdaFormalParameters(); switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case COLON: jj_consume_token(COLON); if (jj_2_33(3, EOF)) { Block(); } else { LambdaBlock(); } break; default: jj_la1[8] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } if (jj_2_33(3, RPAREN)) { Token t2 = jj_consume_token(RPAREN); if (jj_2_33(3, LPAREN)) { Token t = LambdaMethodInvocation(jjtn000.methodName); token = ParserTokenManager.jjTempFillToken(); token.next = t; } else { token = ParserTokenManager.jjTempFillToken(); token.next = t2; } } } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } // map 调用 private void MapInvocation() throws ParseException { TSHMapInvocation jjtn000 = new TSHMapInvocation(T_MapInvocation); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { jj_consume_token(MAP); jj_consume_token(LPAREN); Expression(); label_33: while (true) { if (jj_2_33(3, COMMA)) { jj_consume_token(COMMA); Expression(); } else { break label_33; } } jj_consume_token(RPAREN); } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } // 等号表达式 final public void Assignment() throws ParseException { TSHAssignment jjtn000 = new TSHAssignment(T_Assignment); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); String op; try { PrimaryExpression(); //解析主表达式 op = AssignmentOperator(); //解析操作符 = ,+= ,-= ,*= ,/= ,%= ,>>>= ,== 等等 jjtn000.operator = op; //TSHAssignment保存当前表达式的操作符类型 Expression(); //解析右节点表达式 如 sum(1 ,2 ) , a > b ? a : b 情况太多 } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } // = 操作类型 = , *= ,/= ,%= ,+= ,-= &= ,^= ,<<= ,>>= ,>>>= final public String AssignmentOperator() throws ParseException { Token t; switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case ASSIGN: jj_consume_token(ASSIGN); break; case STARASSIGN: jj_consume_token(STARASSIGN); break; case SLASHASSIGN: jj_consume_token(SLASHASSIGN); break; case MODASSIGN: jj_consume_token(MODASSIGN); break; case PLUSASSIGN: jj_consume_token(PLUSASSIGN); break; case MINUSASSIGN: jj_consume_token(MINUSASSIGN); break; case ANDASSIGN: jj_consume_token(ANDASSIGN); break; case XORASSIGN: jj_consume_token(XORASSIGN); break; case ORASSIGN: jj_consume_token(ORASSIGN); break; case LSHIFTASSIGN: jj_consume_token(LSHIFTASSIGN); break; case RSIGNEDSHIFTASSIGN: jj_consume_token(RSIGNEDSHIFTASSIGN); break; case RUNSIGNEDSHIFTASSIGN: jj_consume_token(RUNSIGNEDSHIFTASSIGN); break; default: jj_la1[24] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } t = getToken(0); { if (true) { return t.kind; } } throw new Error("Missing return statement in function"); } /* boolean a = true; int b = 3; int d = 4; int x =a ? b *=2 :( d*=2);*/ final public void ConditionalExpression() throws ParseException { ConditionalOrExpression(); switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case HOOK: //如果当前是?,那么一定是三目运算符 条件表达式 ? 表达式 : 表达式 jj_consume_token(HOOK); Expression(); //表达式节点解析 jj_consume_token(COLON); TSHTernaryExpression jjtn001 = new TSHTernaryExpression(T_TernaryExpression); boolean jjtc001 = true; jjtree.openNodeScope(jjtn001); jjtreeOpenNodeScope(jjtn001); try { ConditionalExpression(); //条件表达式 } catch (Throwable jjte001) { ... } finally { if (jjtc001) { jjtree.closeNodeScope(jjtn001, 3); jjtreeCloseNodeScope(jjtn001); } } break; default: jj_la1[25] = jj_gen; } } // || 表达式解析 final public void ConditionalOrExpression() throws ParseException { Token t = null; ConditionalAndExpression(); label_7: while (true) { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case BOOL_OR: // || ; break; default: jj_la1[26] = jj_gen; break label_7; } switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case BOOL_OR: t = jj_consume_token(BOOL_OR); break; default: jj_la1[27] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } ConditionalAndExpression(); //三元表达式 TSHBinaryExpression jjtn001 = new TSHBinaryExpression(T_BinaryExpression); boolean jjtc001 = true; jjtree.openNodeScope(jjtn001); jjtreeOpenNodeScope(jjtn001); try { jjtree.closeNodeScope(jjtn001, 2); jjtc001 = false; jjtreeCloseNodeScope(jjtn001); jjtn001.kind = t.kind; } finally { if (jjtc001) { jjtree.closeNodeScope(jjtn001, 2); jjtreeCloseNodeScope(jjtn001); } } } } // && 表达式解析 final public void ConditionalAndExpression() throws ParseException { Token t = null; InclusiveOrExpression(); label_8: while (true) { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case BOOL_AND: // && ; break; default: jj_la1[28] = jj_gen; break label_8; } switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case BOOL_AND: t = jj_consume_token(BOOL_AND); break; default: jj_la1[29] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } InclusiveOrExpression(); TSHBinaryExpression jjtn001 = new TSHBinaryExpression(T_BinaryExpression); boolean jjtc001 = true; jjtree.openNodeScope(jjtn001); jjtreeOpenNodeScope(jjtn001); try { jjtree.closeNodeScope(jjtn001, 2); jjtc001 = false; jjtreeCloseNodeScope(jjtn001); jjtn001.kind = t.kind; } finally { if (jjtc001) { jjtree.closeNodeScope(jjtn001, 2); jjtreeCloseNodeScope(jjtn001); } } } } // | final public void InclusiveOrExpression() throws ParseException { Token t = null; ExclusiveOrExpression(); label_9: while (true) { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case OR: // | ; break; default: jj_la1[30] = jj_gen; break label_9; } switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case OR: t = jj_consume_token(OR); break; default: jj_la1[31] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } ExclusiveOrExpression(); TSHBinaryExpression jjtn001 = new TSHBinaryExpression(T_BinaryExpression); boolean jjtc001 = true; jjtree.openNodeScope(jjtn001); jjtreeOpenNodeScope(jjtn001); try { jjtree.closeNodeScope(jjtn001, 2); jjtc001 = false; jjtreeCloseNodeScope(jjtn001); jjtn001.kind = t.kind; } finally { if (jjtc001) { jjtree.closeNodeScope(jjtn001, 2); jjtreeCloseNodeScope(jjtn001); } } } } // ^ 取反表达式解析 final public void ExclusiveOrExpression() throws ParseException { Token t = null; AndExpression(); label_10: while (true) { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case XOR: ; break; default: jj_la1[32] = jj_gen; break label_10; } t = jj_consume_token(XOR); AndExpression(); TSHBinaryExpression jjtn001 = new TSHBinaryExpression(T_BinaryExpression); boolean jjtc001 = true; jjtree.openNodeScope(jjtn001); jjtreeOpenNodeScope(jjtn001); try { jjtree.closeNodeScope(jjtn001, 2); jjtc001 = false; jjtreeCloseNodeScope(jjtn001); jjtn001.kind = t.kind; } finally { if (jjtc001) { jjtree.closeNodeScope(jjtn001, 2); jjtreeCloseNodeScope(jjtn001); } } } } // & 表达式解析 final public void AndExpression() throws ParseException { Token t = null; EqualityExpression(); label_11: while (true) { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case AND: // & ; break; default: jj_la1[33] = jj_gen; break label_11; } switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case AND: t = jj_consume_token(AND); break; default: jj_la1[34] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } EqualityExpression(); TSHBinaryExpression jjtn001 = new TSHBinaryExpression(T_BinaryExpression); boolean jjtc001 = true; jjtree.openNodeScope(jjtn001); jjtreeOpenNodeScope(jjtn001); try { jjtree.closeNodeScope(jjtn001, 2); jjtc001 = false; jjtreeCloseNodeScope(jjtn001); jjtn001.kind = t.kind; } finally { if (jjtc001) { jjtree.closeNodeScope(jjtn001, 2); jjtreeCloseNodeScope(jjtn001); } } } } // == != final public void EqualityExpression() throws ParseException { Token t = null; RelationalExpression(); label_12: while (true) { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case EQ: // = case NE: // != ; break; default: jj_la1[35] = jj_gen; break label_12; } switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case EQ: t = jj_consume_token(EQ); break; case NE: t = jj_consume_token(NE); break; default: jj_la1[36] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } RelationalExpression(); TSHBinaryExpression jjtn001 = new TSHBinaryExpression(T_BinaryExpression); boolean jjtc001 = true; jjtree.openNodeScope(jjtn001); jjtreeOpenNodeScope(jjtn001); try { jjtree.closeNodeScope(jjtn001, 2); jjtc001 = false; jjtreeCloseNodeScope(jjtn001); jjtn001.kind = t.kind; } finally { if (jjtc001) { jjtree.closeNodeScope(jjtn001, 2); jjtreeCloseNodeScope(jjtn001); } } } } final public void RelationalExpression() throws ParseException { Token t = null; ShiftExpression(); label_13: while (true) { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case GT: // > case LT: // < case LE: // <= case GE: // >= ; break; default: jj_la1[38] = jj_gen; break label_13; } switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case LT: // < t = jj_consume_token(LT); break; case GT: // > t = jj_consume_token(GT); break; case LE: // <= t = jj_consume_token(LE); break; case GE: // >= t = jj_consume_token(GE); break; default: jj_la1[39] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } ShiftExpression(); TSHBinaryExpression jjtn001 = new TSHBinaryExpression(T_BinaryExpression); boolean jjtc001 = true; jjtree.openNodeScope(jjtn001); jjtreeOpenNodeScope(jjtn001); try { jjtree.closeNodeScope(jjtn001, 2); jjtc001 = false; jjtreeCloseNodeScope(jjtn001); jjtn001.kind = t.kind; } finally { if (jjtc001) { jjtree.closeNodeScope(jjtn001, 2); jjtreeCloseNodeScope(jjtn001); } } } } // 位移表达式 << >> >>> final public void ShiftExpression() throws ParseException { Token t = null; AdditiveExpression(); label_14: while (true) { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case LSHIFT: // << case RSIGNEDSHIFT: // >> case RUNSIGNEDSHIFT: // >>> ; break; default: jj_la1[40] = jj_gen; break label_14; } switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case LSHIFT: //<< t = jj_consume_token(LSHIFT); break; case RSIGNEDSHIFT: //>> t = jj_consume_token(RSIGNEDSHIFT); break; case RUNSIGNEDSHIFT: //>>> t = jj_consume_token(RUNSIGNEDSHIFT); break; default: jj_la1[41] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } AdditiveExpression(); TSHBinaryExpression jjtn001 = new TSHBinaryExpression(T_BinaryExpression); boolean jjtc001 = true; jjtree.openNodeScope(jjtn001); jjtreeOpenNodeScope(jjtn001); try { jjtree.closeNodeScope(jjtn001, 2); jjtc001 = false; jjtreeCloseNodeScope(jjtn001); jjtn001.kind = t.kind; } finally { if (jjtc001) { jjtree.closeNodeScope(jjtn001, 2); jjtreeCloseNodeScope(jjtn001); } } } } // + - final public void AdditiveExpression() throws ParseException { Token t = null; MultiplicativeExpression(); label_15: while (true) { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case PLUS: // + case MINUS: // - ; break; default: jj_la1[42] = jj_gen; break label_15; } switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case PLUS: // + t = jj_consume_token(PLUS); break; case MINUS: // - t = jj_consume_token(MINUS); break; default: jj_la1[43] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } MultiplicativeExpression(); TSHBinaryExpression jjtn001 = new TSHBinaryExpression(T_BinaryExpression); boolean jjtc001 = true; jjtree.openNodeScope(jjtn001); jjtreeOpenNodeScope(jjtn001); try { jjtree.closeNodeScope(jjtn001, 2); jjtc001 = false; jjtreeCloseNodeScope(jjtn001); jjtn001.kind = t.kind; } finally { if (jjtc001) { jjtree.closeNodeScope(jjtn001, 2); jjtreeCloseNodeScope(jjtn001); } } } } // / % 处理 final public void MultiplicativeExpression() throws ParseException { Token t = null; UnaryExpression(); label_16: while (true) { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case STAR: // * case SLASH: // / case MOD: // % ; break; default: jj_la1[44] = jj_gen; break label_16; } switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case STAR: // * t = jj_consume_token(STAR); break; case SLASH: // / t = jj_consume_token(SLASH); break; case MOD: // % t = jj_consume_token(MOD); break; default: jj_la1[45] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } UnaryExpression(); //二元表达式处理 TSHBinaryExpression jjtn001 = new TSHBinaryExpression(T_BinaryExpression); boolean jjtc001 = true; jjtree.openNodeScope(jjtn001); jjtreeOpenNodeScope(jjtn001); try { jjtree.closeNodeScope(jjtn001, 2); jjtc001 = false; jjtreeCloseNodeScope(jjtn001); jjtn001.kind = t.kind; } finally { if (jjtc001) { jjtree.closeNodeScope(jjtn001, 2); jjtreeCloseNodeScope(jjtn001); } } } } // 一元表达式处理 final public void UnaryExpression() throws ParseException { Token t = null; switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case PLUS: // + case MINUS: // - switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case PLUS: // ++ t = jj_consume_token(PLUS); break; case MINUS: // -- t = jj_consume_token(MINUS); break; default: jj_la1[46] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } // 一元表达式 UnaryExpression(); TSHUnaryExpression jjtn001 = new TSHUnaryExpression(T_UnaryExpression); boolean jjtc001 = true; jjtree.openNodeScope(jjtn001); jjtreeOpenNodeScope(jjtn001); try { jjtree.closeNodeScope(jjtn001, 1); jjtc001 = false; jjtreeCloseNodeScope(jjtn001); jjtn001.kind = t.kind; } finally { if (jjtc001) { jjtree.closeNodeScope(jjtn001, 1); jjtreeCloseNodeScope(jjtn001); } } break; case INCR: // ++ i PreIncrementExpression(); break; case DECR: // -- i PreDecrementExpression(); break; case BOOL: case FALSE: case NULL: case TRUE: case IDENTIFIER: case STR: case NUMBER: case LPAREN: case BANG: case TILDE: case NEW: // 非 + ,- 的一元表达式 UnaryExpressionNotPlusMinus(); break; default: jj_la1[47] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } } // ++ final public void PreIncrementExpression() throws ParseException { Token t = null; t = jj_consume_token(INCR); PrimaryExpression(); TSHUnaryExpression jjtn001 = new TSHUnaryExpression(T_UnaryExpression); boolean jjtc001 = true; jjtree.openNodeScope(jjtn001); jjtreeOpenNodeScope(jjtn001); try { jjtree.closeNodeScope(jjtn001, 1); jjtc001 = false; jjtreeCloseNodeScope(jjtn001); jjtn001.kind = t.kind; } finally { if (jjtc001) { jjtree.closeNodeScope(jjtn001, 1); jjtreeCloseNodeScope(jjtn001); } } } final public void PreDecrementExpression() throws ParseException { Token t = null; t = jj_consume_token(DECR); PrimaryExpression(); TSHUnaryExpression jjtn001 = new TSHUnaryExpression(T_UnaryExpression); boolean jjtc001 = true; jjtree.openNodeScope(jjtn001); jjtreeOpenNodeScope(jjtn001); try { jjtree.closeNodeScope(jjtn001, 1); jjtc001 = false; jjtreeCloseNodeScope(jjtn001); jjtn001.kind = t.kind; } finally { if (jjtc001) { jjtree.closeNodeScope(jjtn001, 1); jjtreeCloseNodeScope(jjtn001); } } } // ~ ! final public void UnaryExpressionNotPlusMinus() throws ParseException { Token t = null; switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case BANG: case TILDE: switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case TILDE: t = jj_consume_token(TILDE); // ~ break; case BANG: t = jj_consume_token(BANG); // ! break; default: jj_la1[48] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } UnaryExpression(); TSHUnaryExpression jjtn001 = new TSHUnaryExpression(T_UnaryExpression); boolean jjtc001 = true; jjtree.openNodeScope(jjtn001); jjtreeOpenNodeScope(jjtn001); try { jjtree.closeNodeScope(jjtn001, 1); jjtc001 = false; jjtreeCloseNodeScope(jjtn001); jjtn001.kind = t.kind; } finally { if (jjtc001) { jjtree.closeNodeScope(jjtn001, 1); jjtreeCloseNodeScope(jjtn001); } } break; default: jj_la1[49] = jj_gen; switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case FALSE: case NULL: case TRUE: case STR: case IDENTIFIER: case NUMBER: case LPAREN: case NEW: PostfixExpression(); //后缀表达式 break; default: jj_la1[50] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } } } // ++ -- final public void PostfixExpression() throws ParseException { Token t = null; if (jj_2_12(2147483647)) { PrimaryExpression(); switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case INCR: // i ++ t = jj_consume_token(INCR); break; case DECR: // i -- t = jj_consume_token(DECR); break; default: jj_la1[53] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } TSHUnaryExpression jjtn001 = new TSHUnaryExpression(T_UnaryExpression); boolean jjtc001 = true; jjtree.openNodeScope(jjtn001); jjtreeOpenNodeScope(jjtn001); try { jjtree.closeNodeScope(jjtn001, 1); jjtc001 = false; jjtreeCloseNodeScope(jjtn001); jjtn001.kind = t.kind; jjtn001.postfix = true; } finally { if (jjtc001) { jjtree.closeNodeScope(jjtn001, 1); jjtreeCloseNodeScope(jjtn001); } } } else { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case FALSE: case NULL: case TRUE: case STR: case NUMBER: case IDENTIFIER: case LPAREN: case NEW: PrimaryExpression(); break; default: jj_la1[54] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } } } // { [ . final public void PrimaryExpression() throws ParseException { TSHPrimaryExpression jjtn000 = new TSHPrimaryExpression(T_PrimaryExpression); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { PrimaryPrefix(); // 前缀表达式处理 label_17: while (true) { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case LBRACE: // { case LBRACKET: // [ case DOT: // . ; break; default: jj_la1[56] = jj_gen; break label_17; } PrimarySuffix(); //后缀表达式 } } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } final public void PrimaryPrefix() throws ParseException { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case FALSE: case NULL: case TRUE: case NUMBER: case STR: Literal(); // false ,null ,true ,number ,str 常量表达式 break; case LPAREN: // ( a + b ) 的情况 jj_consume_token(LPAREN); // 为表达式去括号 Expression(); jj_consume_token(RPAREN); break; case NEW: AllocationExpression(); // new Exception()情况 break; default: jj_la1[57] = jj_gen; if (jj_2_14(2147483647)) { //方法调用 ,如 sum(1,2) MethodInvocation(); } else { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case IDENTIFIER: // 变量名 a = 1 ,此时 a 是变量名,需要从名称空间中取变量的 AmbiguousName(); break; default: jj_la1[58] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } } } } // lambda 表达式方法调用 final public Token LambdaMethodInvocation(String methodName) throws ParseException { TSHMethodInvocation jjtn000 = new TSHMethodInvocation(T_MethodInvocation); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); Token t = null; try { MethodInvocation() (methodName); //方法名称 while (jj_2_33(3, LPAREN)) { //如果存在 a()()形式的方法调用 t = Arguments(); } } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } return t; } // 方法调用 final public void MethodInvocation() throws ParseException { TSHMethodInvocation jjtn000 = new TSHMethodInvocation(T_MethodInvocation); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { AmbiguousName(); //方法名称 while (jj_2_33(3, LPAREN)) { //如果存在 a()()形式的方法调用 Arguments(); } } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } final public void AmbiguousName(String name) { TSHAmbiguousName jjtn000 = new TSHAmbiguousName(T_AmbiguousName); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); jjtn000.text = name; } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } //变量名,或方法引用名 final public void AmbiguousName() throws ParseException { TSHAmbiguousName jjtn000 = new TSHAmbiguousName(T_AmbiguousName); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); Token t; StringBuffer s; try { t = jj_consume_token(IDENTIFIER); s = new StringBuffer(t.image); label_5: while (true) { if (jj_2_7(2)) { ; } else { break label_5; } jj_consume_token(DOT); t = jj_consume_token(IDENTIFIER); s.append("." + t.image); } jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); jjtn000.text = s.toString(); } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } // 自定义字面量节点 final public void CustomLiteral(String kind, Object value) throws ParseException { TSHLiteral jjtn000 = new TSHLiteral(T_Literal); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); boolean b; String literal = value.toString(); try { switch (kind) { case NUMBER: jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); TBigDecimal tBigDecimal = new TBigDecimal(); tBigDecimal.setValue(TNumberUtil.objToBigDecimalDefault(literal, BigDecimal.ZERO)); tBigDecimal.setPrecision(TNumberUtil.getPrecision(literal)); try { jjtn000.value = tBigDecimal; } catch (NumberFormatException e) { { if (true) throw createParseException( "Error or number too big for integer type: " + literal); } } break; case STR: jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); try { jjtn000.stringSetup(literal.substring(1, literal.length() - 1)); } catch (Exception e) { { if (true) throw createParseException("Error parsing string: " + literal); } } break; case FALSE: case TRUE: b = TNumberUtil.objToBooleanDefault(value, false); jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); jjtn000.value = b ? Primitive.TRUE : Primitive.FALSE; break; case NULL: jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); jjtn000.value = Primitive.NULL; break; default: jj_la1[61] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } final public void Literal() throws ParseException { TSHLiteral jjtn000 = new TSHLiteral(T_Literal); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); Token x; boolean b; String literal; try { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case NUMBER: //如果是 number 类型,创建 TBigDecimal 对象,这个对象中值为BigDecimal 类型,精度为小数点后几位 //在 java中 int类型的精度为0,值为 int类型的具体值,在 java中所有的数值类型都可以用TBigDecimal来封装 x = jj_consume_token(NUMBER); jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); literal = x.image; TBigDecimal tBigDecimal = new TBigDecimal(); tBigDecimal.setValue(TNumberUtil.objToBigDecimalDefault(literal, BigDecimal.ZERO)); tBigDecimal.setPrecision(TNumberUtil.getPrecision(literal)); try { jjtn000.value = tBigDecimal; } catch (NumberFormatException e) { { if (true) throw createParseException( "Error or number too big for integer type: " + literal); } } break; case STR: //字符串类型, x = jj_consume_token(STR); jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); try { //字符串类型,处理掉字符串中的转化义字符 jjtn000.stringSetup(x.image.substring(1, x.image.length() - 1)); } catch (Exception e) { { if (true) throw createParseException("Error parsing string: " + x.image); } } break; case FALSE: //true ,false 表示 boolean 类型字面量 case TRUE: b = BooleanLiteral(); jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); jjtn000.value = b ? Primitive.TRUE : Primitive.FALSE; break; case NULL: //空类型 NullLiteral(); jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); jjtn000.value = Primitive.NULL; break; default: jj_la1[61] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } final public boolean BooleanLiteral() throws ParseException { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case TRUE: jj_consume_token(TRUE); { if (true) return true; } break; case FALSE: jj_consume_token(FALSE); { if (true) return false; } break; default: jj_la1[62] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } throw new Error("Missing return statement in function"); } final public void NullLiteral() throws ParseException { jj_consume_token(NULL); } //参数列表 final public Token Arguments() throws ParseException { TSHArguments jjtn000 = new TSHArguments(T_Arguments); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); Token t = null; try { jj_consume_token(LPAREN); switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case FALSE: case NULL: case TRUE: case IDENTIFIER: case STR: case NUMBER: case LPAREN: case LBRACKET: case LBRACE: case BANG: case TILDE: case INCR: case DECR: case PLUS: case MINUS: case STAR: case SSTAR: case LAMBDA: case MAP: ArgumentList(); break; default: jj_la1[63] = jj_gen; ; } t = jj_consume_token(RPAREN); } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } return t; } // leave these on the stack for Arguments() to handle final public void ArgumentList() throws ParseException { Expression(); label_18: while (true) { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case COMMA: ; break; default: jj_la1[64] = jj_gen; break label_18; } jj_consume_token(COMMA); Expression(); } } // new 表达式解析 final public void AllocationExpression() throws ParseException { TSHAllocationExpression jjtn000 = new TSHAllocationExpression(T_AllocationExpression); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); try { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case NEW: jj_consume_token(NEW); //变量名 AmbiguousName(); switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case LPAREN: Arguments(); //方法参数 new A(a,b,c)的情况 break; default: jj_la1[65] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } break; default: jj_la1[66] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } final public void PrimarySuffix() throws ParseException { TSHPrimarySuffix jjtn000 = new TSHPrimarySuffix(T_PrimarySuffix); boolean jjtc000 = true; jjtree.openNodeScope(jjtn000); jjtreeOpenNodeScope(jjtn000); Token t = null; try { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case LBRACKET: jj_consume_token(LBRACKET); int flag = jj_3_41(33); if (flag == 1) { // [:3] CustomLiteral(NUMBER, 0); jj_consume_token(COLON); Expression(); } else if (flag == 2) { // [1:] Expression(); jj_consume_token(COLON); CustomLiteral(NUMBER, DEFAULT_INT_MAX); //默认最大值为 int最大值 } else if (flag == 3) { // [1:3] Expression(); jj_consume_token(COLON); Expression(); } else { // [3] Expression(); } jj_consume_token(RBRACKET); jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); jjtn000.operation = TSHPrimarySuffix.INDEX; break; case DOT: jj_consume_token(DOT); t = jj_consume_token(IDENTIFIER); switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case LPAREN: Arguments(); break; default: jj_la1[59] = jj_gen; ; } jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); jjtn000.operation = TSHPrimarySuffix.NAME; jjtn000.field = t.image; break; case LBRACE: jj_consume_token(LBRACE); Expression(); jj_consume_token(RBRACE); jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; jjtreeCloseNodeScope(jjtn000); jjtn000.operation = TSHPrimarySuffix.PROPERTY; break; default: jj_la1[60] = jj_gen; jj_consume_token(default_1); throw new ParseException(); } } catch (Throwable jjte000) { ... } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); jjtreeCloseNodeScope(jjtn000); } } } final public boolean Line() throws ParseException { switch ((jj_ntk == default_1) ? jj_ntk() : jj_ntk) { case EOF: jj_consume_token(EOF); Interpreter.debug("End of File!"); { if (true) return true; } break; default: jj_la1[0] = jj_gen; return !BlockStatement(); } throw new Error("Missing return statement in function"); }
在创建执行树的过程中,我们需要注意下面几个方法,首先,最常用的jj_consume_token方法
final private Token jj_consume_token(String kind) throws ParseException { Token oldToken; if ((oldToken = token).next != null) { token = token.next; } else { token = token.next = token_source.getNextToken(); } jj_ntk = default_1; if (eq(token.kind, kind)) { jj_gen++; if (++jj_gc > 100) { jj_gc = 0; for (int i = 0; i < jj_2_rtns.length; i++) { JJCalls c = jj_2_rtns[i]; while (c != null) { if (c.gen < jj_gen) { c.first = null; } c = c.next; } } } return token; } token = oldToken; jj_kind = kind; String message = token.beginLine + "行," + token.beginColumn + "列,代码书写有误"; throw new ParseException(message); }
这个方法的意思是当传入 for类型的 kind 时,此时当前token的next一定是for类型,如果不是则抛出异常,如果当前 token的next 为空,则调用 getNextToken()方法,向文件中再读取一个Token。
public class JJTParserState { private java.util.Stack nodes; private java.util.Stack marks; private int sp; private int mk; private boolean node_created; public JJTParserState() { nodes = new java.util.Stack(); marks = new java.util.Stack(); sp = 0; mk = 0; } public void reset() { nodes.removeAllElements(); marks.removeAllElements(); sp = 0; mk = 0; } public Node rootNode() { return (Node) nodes.elementAt(0); } void pushNode(Node n) { nodes.push(n); ++sp; } public Node popNode() { if (--sp < mk) { mk = ((Integer) marks.pop()).intValue(); } return (Node) nodes.pop(); } public Node peekNode() { return (Node) nodes.peek(); } public int nodeArity() { return sp - mk; } //出现异常时调用 public void clearNodeScope(Node n) { while (sp > mk) { popNode(); } mk = ((Integer) marks.pop()).intValue(); } public void openNodeScope(Node n) { marks.push(new Integer(mk)); mk = sp; n.jjtOpen(); } /* 一个确定的节点由指定数量的子代构成。从堆栈中弹出该数量的节点,并将其作为确定节点的子代。然后将确定的节点压入堆栈。 */ public void closeNodeScope(Node n, int num) { mk = ((Integer) marks.pop()).intValue(); while (num-- > 0) { Node c = popNode(); c.jjtSetParent(n); n.jjtAddChild(c, num); } n.jjtClose(); pushNode(n); node_created = true; } /* 如果条件为真,则构造条件节点。自从节点打开以来,所有已推入的节点成为条件节点的子代, 然后将其推入堆栈。如果条件为假,则不构造该节点,并且将它们保留在堆栈中。*/ public void closeNodeScope(Node n, boolean condition) { if (condition) { int a = nodeArity(); mk = ((Integer) marks.pop()).intValue(); while (a-- > 0) { Node c = popNode(); c.jjtSetParent(n); n.jjtAddChild(c, a); } n.jjtClose(); pushNode(n); node_created = true; } else { mk = ((Integer) marks.pop()).intValue(); node_created = false; } } }
closeNodeScope方法是创建树的关键代码,有重载的两个方法closeNodeScope(Node n, boolean condition) ,closeNodeScope(Node n, int num) ,这两个方法有什么区别呢?
先来说方法1 ,如 Stack中至顶向下分别有 c,b,a 三个元素,此时传入d,
则会创建一棵树
再来看方法2,同样栈中有3个元素,自顶向下c,b,a , 调用closeNodeScope(Node d, 2)
通过上述两个方法分析,我相信大家对树的创建过程己经有了深刻的理解了。
下面是关于 a = 3 被解析成执行树的结构。
下面是关于
for(i in range(5)){
print(i)
}
的树结构
经过 Parser类,得到了一棵执行树,下面,我们对执行树的某些节点来做解析。
执行树节点分析
SimpleNode 类中有一个 eval()方法,获取节点值时调用该方法。这么多节点,也不好一个一个去解析,选几个来进行解析吧,先看TSHForStatement,也就是 for 循环的执行节点。
for结点解析
TSHForStatement.java
public class TSHForStatement extends SimpleNode implements ParserConstants { public String kOrI; //可能是 map 的 key ,也可能是 list 的索引 public String varName; //变量名 private String label; public TSHForStatement(String id) { super(id); } public Object eval(CallStack callstack, Interpreter interpreter, String label) throws EvalError { this.label = label; return eval(callstack, interpreter); } // 下面针对 for( i, item in [1,2,3,4,5]){print(i,item)} 和 for (i in range(5)){print(i)} 两种情况来分析 public Object eval(CallStack callstack, Interpreter interpreter) throws EvalError { Class elementType = null; SimpleNode expression; SimpleNode statement = null; NameSpace enclosingNameSpace = callstack.top(); SimpleNode firstNode = ((SimpleNode) jjtGetChild(0)); int nodeCount = jjtGetNumChildren(); expression = firstNode; // for 循环的第一个节点是表达式,range(5) if (nodeCount > 1) { statement = ((SimpleNode) jjtGetChild(1)); } //定义for循环的命名空间 BlockNameSpace eachNameSpace = new BlockNameSpace(enclosingNameSpace); callstack.swap(eachNameSpace); //获取for表达式的迭代元素 final Object iteratee = expression.eval(callstack, interpreter); if (iteratee == Primitive.NULL) { throw new EvalError("The collection, array, map, iterator, or " + "enumeration portion of a for statement cannot be null.", this, callstack); } CollectionManager cm = CollectionManager.getCollectionManager(); if (!cm.isBshIterable(iteratee)) { if (iteratee instanceof Map) { //如果迭代器是 Map 类型,则调用 map遍历的方法 //对于 map 的处理 return forMap(enclosingNameSpace, eachNameSpace, (Map) iteratee, statement, callstack, interpreter); } throw new EvalError("Can't iterate over type: " + iteratee.getClass(), this, callstack); } BshIterator iterator = cm.getBshIterator(iteratee); Object returnControl = Primitive.VOID; int i = 0; while (iterator.hasNext()) { // iterator遍历 try { Object value = iterator.next(); if (value == null) { value = Primitive.NULL; } //如果 for循环是这样写的 for(i ,item in [1,2,3]){...} //i 表示迭代的索引 0 ,1,2 ,item 表示迭代的内容 1,2,3 if (TStringUtil.isNotBlank(kOrI)) { eachNameSpace.setVariable(kOrI, i, false); } if (elementType != null) { // 设置局部变量 item eachNameSpace.setTypedVariable(varName/*name*/, elementType/*type*/, value/*value*/, null/*none*/); } else { eachNameSpace.setVariable(varName, value, false); } } catch (UtilEvalError e) { throw e.toEvalError("for loop iterator variable:" + varName, this, callstack); } boolean breakout = false;// switch eats a multi-level break here? if (statement != null) { Object ret = statement.eval(callstack, interpreter); // 执行 {} 内的内容 if (ret instanceof ReturnControl) { //如果返回结果是return 类型的 String retLabel = ((ReturnControl) ret).label; if (TStringUtil.isNotBlank(retLabel)) { //处理 break label 的情况 if (!retLabel.equals(label)) { returnControl = ret; } breakout = true; } else { switch (((ReturnControl) ret).kind) { //普通返回的处理 case RETURN: //如果是return ,则终止所有迭代 returnControl = ret; breakout = true; break; case CONTINUE: //如果是continue继续迭代 break; case BREAK: //如果是break,终止当前迭代 breakout = true; break; } } } } if (breakout) { break; } i++; } callstack.swap(enclosingNameSpace); return returnControl; } //对 map 进行遍历 public Object forMap(NameSpace enclosingNameSpace, BlockNameSpace eachNameSpace, Map<Object, Object> map, SimpleNode statement, CallStack callstack, Interpreter interpreter) throws EvalError { Object returnControl = Primitive.VOID; for (Map.Entry<Object, Object> entry : map.entrySet()) { try { //设置 map 的 key 和 value eachNameSpace.setVariable(kOrI, entry.getKey(), false); eachNameSpace.setVariable(varName, entry.getValue(), false); } catch (UtilEvalError e) { throw e.toEvalError("for loop iterator variable:" + varName, this, callstack); } // switch eats a multi-level break here? boolean breakout = false; if (statement != null) { Object ret = statement.eval(callstack, interpreter);// {} 执行 if (ret instanceof ReturnControl) { String retLabel = ((ReturnControl) ret).label; //处理 break label 的情况 if (TStringUtil.isNotBlank(retLabel)) { if (!retLabel.equals(label)) { returnControl = ret; } breakout = true; } else { switch (((ReturnControl) ret).kind) { case RETURN: // return 返回多层循环 returnControl = ret; breakout = true; break; case CONTINUE: // continue break; case BREAK: // break breakout = true; break; } } } } if (breakout) { break; } } callstack.swap(enclosingNameSpace); return returnControl; } }
for 循环的遍历其实代码实现也是非常简单的,先创建 for循环的NameSpace,创建 NameSpace 的好处是,for 能对外层变量访问修改,但是 for 循环内部定义的变量不会污染循环之外的变量,定义名称空间之后,将循环迭代的变量设置到 NameSpace 中,此时执行 for 循环内的 Block 模块,Block内执行就能访问 NameSpace定义的迭代变量了,当Block执行完毕,判断当前返回值的类型,如果是 Return或 Break,则终止本次循环,如果是CONTINUE,则继续循环,但需要注意的是break 只是终止本次循环,而 return会将 ret 结果值继续返回上一层调用。如果上一层也是循环迭代,会继续向上一层返回。
但是上述中还需要注意的一点是 beak label:的实现,我们先来看看一个简单示例。
label_1: for(i in range(1,10)){ // @1 for(j in range(1,10)){ // @2 if(j > 5 ){ break label_1 } print(i,j) } }
执行结果:
这个功能又是如何实现的呢?当解析到外层循环@1 时,会将发现外层循环@1上有label_1:标签,因此将 label_1保存到外层循环label属性中,当解析到 break时,发现后面接了 label_1,因些将将 label_1保存到ReturnControl的retLabel属性中,当if 条件符合break label_1时,将返回带retLabel属性值的ReturnControl对象,就有了下面代码判断了,如果retLabel值等于当前循环的label值,则终止本次循环,并且不再将returnControl值返回,如果不相等,则继续向外层循环抛出returnControl,并终止本次循环。
if (TStringUtil.isNotBlank(retLabel)) { if (!retLabel.equals(label)) { returnControl = ret; } breakout = true; }
for循环实现如此,当然while循环实现亦是如此。
lable_1: while(true){ switch(b){ case 5 : break lable_1 default: print(b) break; } b ++ }
方法定义结点解析
TSHMethodDeclaration.java
public class TSHMethodDeclaration extends SimpleNode { public String methodName;//方法名称 //方法的参数节点 public TSHFormalParameters paramsNode; public int firstThrowsClause; //方法内{...}块内容 public TSHBlock blockNode; // throw exception1,exception2 异常类个数 public int numThrows = 0; public TSHMethodDeclaration(String id) { super(id); } public synchronized void insureNodesParsed() { if (paramsNode != null) { return; } //初始化方法的参数节点 paramsNode = (TSHFormalParameters) jjtGetChild(0); //初始化方法的块节点 blockNode = (TSHBlock) jjtGetChild(1 + numThrows); // skip throws } public Object eval(CallStack callstack, Interpreter interpreter) throws EvalError { return eval(callstack, interpreter, null); } public Object eval(CallStack callstack, Interpreter interpreter, TSHMethodInvocation invocation) throws EvalError { evalNodes(callstack, interpreter); NameSpace namespace = callstack.top(); NameSpace methodNamespace = namespace; BshClassManager bcm = namespace.getClassManager(); //如果当前命名空间是全局命名空间,自定义方法命名空间,防止变量污染 if ("global".equals(namespace.getName())) { methodNamespace = new NameSpace(bcm, "self_" + methodName); methodNamespace.methods = namespace.methods; } //方法所携带的数据信息,如方法添加了@logger注解信息 TMethodData methodData = new TMethodData(invocation); //定义method TshMethod bshMethod = new TshMethod(this, methodNamespace, methodData); try { //将方法保存到当前命名空间 namespace.setMethod(methodName, bshMethod); } catch (UtilEvalError e) { throw e.toEvalError(this, callstack); } return bshMethod; } public void evalNodes(CallStack callstack, Interpreter interpreter) throws EvalError { insureNodesParsed(); for (int i = firstThrowsClause; i < numThrows + firstThrowsClause; i++) { ((TSHAmbiguousName) jjtGetChild(i)).toClass(callstack, interpreter); } paramsNode.eval(callstack, interpreter); } public String toString() { return "MethodDeclaration: " + id; } }
为什么当前名称空间是global时,需要为当前方法创建独立的名称空间呢?但方法却引用了global 名称空间的方法呢。这是为什么呢?
def add(x, y){ return x + y } def logger(fn){ def _logger(x,y){ print('before', x,y) ret = fn(x,y) print('end ', ret) return ret } return _logger } t = logger(add) t(4,9)
上述脚本中,当调用 logger方法时,将add方法传入,此时add方法存储于logger的 NameSpace 中,并将_logger()方法引用返回,并存储于global的NameSpace 中,当调用 t(4,9)时,此时调用的是之前存储的 _logger()方法,当执行到 fn(x,y)时,fn 在 _logger()方法的NameSpace中并没有,因此会向上一层方法的 NameSpace中寻找,正好,因为之前调用过logger(add)方法,将 add 方法存储在了 logger()方法的 NameSpace中,因此,找到了fn所代表的 add 方法,执行并返回,同理,变量也可以向外层 NameSpace 中寻找,但是问题来了。请看下例
a = 100 def sum(){ a = 1 } print(a)
此时打印出 a = 1 ,显然不是我们想要的,方法内部变量应该是一个独立的名称空间,不能对全局变量污染。
def add (x,y){ return x + y } def sum(){ return add(1,2) } print(sum())
方法的内部能对外部方法进行访问,当方法外部的NameSpace是global 时,需要将方法内部变量和外部变量隔离,从而防止变量污染。
方法调用结点解析
public class TSHMethodInvocation extends SimpleNode { public TSHMethodInvocation(String id) { super(id); } TSHAmbiguousName getNameNode() { return (TSHAmbiguousName) jjtGetChild(0); } TSHArguments getArgsNode(int i) { return (TSHArguments) jjtGetChild(i); } public Object eval(CallStack callstack, Interpreter interpreter) throws EvalError { NameSpace namespace = callstack.top(); TSHAmbiguousName nameNode = getNameNode(); if (namespace.getParent() != null && namespace.getParent().isClass && (nameNode.text.equals("super") || nameNode.text.equals("this"))) { return Primitive.VOID; } Name name = nameNode.getName(namespace); try { Object result = null; TshMethod method = namespace.getMethod(nameNode.text.trim(), new Class[]{null}); // 如果有注解方法调用 if (method != null && method.methodData != null && method.methodData.invocation != null) { Object[] args = this.getArgsNode(1).getArguments(callstack, interpreter); result = invoke(method, args, callstack, interpreter, false); } else { //处理 a(1) 和 a(1)(2)...(3)类型的方法调用 for (int i = 1; i < jjtGetNumChildren(); i++) { Object[] args = getArgsNode(i).getArguments(callstack, interpreter); if (i == 1) { result = name.invokeMethod(interpreter, args, callstack, this); } else { result = ((TshMethod) result).invokeNew(args, interpreter, callstack, this); } } } return result; } catch (ReflectError e) { throw new EvalError("Error in method invocation: " + e.getMessage(), this, callstack); } catch (InvocationTargetException e) { String msg = "Method Invocation " + name; Throwable te = e.getTargetException(); boolean isNative = true; if (te instanceof EvalError) if (te instanceof TargetError) isNative = ((TargetError) te).inNativeCode(); else isNative = false; throw new TargetError(msg, te, this, callstack, isNative); } catch (UtilEvalError e) { throw e.toEvalError(this, callstack); } } public Object invoke(TshMethod method, Object[] argValues, CallStack callstack, Interpreter interpreter, boolean flag) throws EvalError, UtilEvalError { Object result = null; if (method != null && method.methodData != null && method.methodData.invocation != null) { // 如果有注解方法调用 TSHMethodInvocation methodInvocation = method.methodData.invocation; method.methodData = null; // 防止递归调用 String methodName = getMethodName(methodInvocation); List<String> aMethodNames = new ArrayList<>(); List<String> aVariableNames = new ArrayList<>(); try { NameSpace namespace = new NameSpace(callstack.top(), methodName); callstack.push(namespace); result = namespace.getMethod(methodName, new Class[]{null}); for (int i = 0; i < methodInvocation.jjtGetNumChildren(); i++) { if (i == methodInvocation.jjtGetNumChildren() - 1) { //将方法本身作为参数传入 result = invoke(((TshMethod) result), new Object[]{method}, callstack, interpreter, true); //清除掉注解调用方法产生的局部变量和局部方法 removeNotNeed(callstack.top(), aMethodNames, aVariableNames); } else { Object[] args = methodInvocation.getArgsNode(i + 1).getArguments(callstack, interpreter); result = invoke(((TshMethod) result), args, callstack, interpreter, true); removeNotNeed(callstack.top(), aMethodNames, aVariableNames); } } //调用实际调用的方法 result = invoke(((TshMethod) result), argValues, callstack, interpreter, true); } catch (UtilEvalError utilEvalError) { utilEvalError.printStackTrace(); } catch (EvalError evalError) { evalError.printStackTrace(); } finally { //防止递归调用,先设置为空,再恢复 method.methodData = new TMethodData(methodInvocation); NameSpace pop = callstack.pop(); if (flag) { NameSpace top = callstack.top(); for (String name : pop.getVariableNames()) { if (aVariableNames.contains(name)) { continue; } Object value = pop.getVariable(name); Object old = top.getVariable(name); top.setVariable(name, old == null || old == Primitive.VOID ? value : old, false); } for (String name : pop.getMethodNames()) { if (aMethodNames.contains(name)) { continue; } TshMethod value = pop.getMethod(name, new Class[]{null}); TshMethod old = top.getMethod(name, new Class[]{null}); value = old == null ? value : old; if (value != null) { top.setMethod(name, value); } } } } return result; } else { result = method.invokeNew(argValues, interpreter, callstack, this, true); } return result; } public void removeNotNeed(NameSpace parent, List<String> methodNames, List<String> variableNames) { for (String name : parent.getVariableNames()) { variableNames.add(name); } for (String name : parent.getMethodNames()) { methodNames.add(name); } } public String getMethodName(TSHMethodInvocation methodInvocation) { TSHAmbiguousName annotaionName = methodInvocation.getNameNode();//注解名称 return annotaionName.text.trim(); } }
上述源码中,可能大家看起来比较晕,那我们先来了解方法调用的四种情况
第一种 def max(a ,b ){ return a > b ? a : b } print(max(1,30)) 第二种情况 def sum(x){ def add(y,z){ return x + y + z } return add } a = sum(1)(2,3) print(a) 第三种 def add(x, y){ return x + y } def logger(fn){ def _logger(x,y){ print('before', x,y) ret = fn(x,y) print('end ', ret) return ret } return _logger } t = logger(add) t(4,9) 第四种情况 def copy_properties(src){ def inner(dest){ print(src) print(dest) return dest } return inner } def log(mm){ print('*'*30,mm) return mm } @log def logger(exet, func){ @log def yy(){ @log def xx(vv){ @log def _logger(fn){ @copy_properties(func) def wrapper(*args, **kwargs){ print(vv) print('mmmmmm',args) print('xxxxxxxx',exet) ret = fn(*args, **kwargs) print('nnnnnnnnnnnnnnnnnnnn',fn) print(m) print('xxxxxxxxxx') func(*args, **kwargs) return ret } m = 'wwwwwwwwwwwwwwwwwwww' return wrapper } return _logger } return xx } return yy } def b (){ x = 1 return x } @logger(2000,b)(10)(33) def add(x ,y){ print('.........',x,y) return x + y } print(add(4,5))
第一种情况是我们最常用的场景,在我们源码中只是调用invokeMethod即可,第二种情况也不难,直接将第一种情况返回值作为方法调用即可,第三种情况,即简单的柯里化,invoke()方法即专门处理第四种情况的。来看一下第四种情况的执行结果。
对于第四种情况写法我觉得不是很优雅,但目前我也没找到更好的解决办法,本想给每一个柯里化方法分配一个NameSpace,但发现程序走着走着变量,变量丢失了,因为个人时间问题,没有太多的去研究了,感兴趣的小伙伴可以想一下,如何写得更好。
方法的内部执行是调用了 invokeNew 方法,接下来,看看内部实现。
public Object invokeNew(Object[] argValues, Interpreter interpreter, CallStack callstack, SimpleNode callerInfo) throws EvalError { return invokeImplNew(argValues, interpreter, callstack, callerInfo, false); } private Object invokeImplNew(Object[] argValues, Interpreter interpreter, CallStack callstack, SimpleNode callerInfo, boolean overrideNameSpace) throws EvalError { Class returnType = getReturnType(); if (callstack == null) callstack = new CallStack(declaringNameSpace); if (argValues == null) argValues = new Object[]{}; NameSpace localNameSpace; if (overrideNameSpace) { localNameSpace = callstack.top(); } else { localNameSpace = new NameSpace(declaringNameSpace, name); localNameSpace.isMethod = true; } localNameSpace.setNode(callerInfo); //对方法参数进行排序 如,方法参数定义为 method(x,y,*args,**kwargs,z), //排成 method(x,y,z,*args,**kwargs),将包含*的参数放到倒数第二们, //**参数放到倒数第一位 List<TVar> dVs = getSortArgValues(defaultValues); List<Object> args = arrayToList(argValues); //下面主要是对方法参数赋值了,如果方法调用没有传递参数 ,则使用默认值, //先根据方法参数名匹配,再根据位置进行匹配,最后没有匹配的参数分别被 //*args 和**kwargs 吃掉 for (int i = 0; i < numArgs; i++) { try { String paramName = dVs.get(i).getName(); Object argValue = null; TVar tVar = dVs.get(i); if (Utils.eq(tVar.getKind(), STAR)) { List<Object> list = new ArrayList<>(); for (int j = 0; j < args.size(); j++) { Object o = args.get(j); if (!(o instanceof TVar)) { list.add(o); } } argValue = list; } else if (Utils.eq(tVar.getKind(), SSTAR)) { Map<String, Object> map = new LinkedHashMap<>(); for (Object o : args) { if (o instanceof TVar) { map.put(((TVar) o).getName(), ((TVar) o).getValue()); } } argValue = map; } else { int k = 0; for (int j = 0; j < args.size(); j++) { Object o = args.get(j); if (o instanceof TVar) { if (Utils.eq(((TVar) o).getName(), paramName)) { argValue = ((TVar) o).getValue(); k = j; break; } } } if (argValue == null) { for (int j = 0; j < args.size(); j++) { Object o = args.get(j); if (!(o instanceof TVar)) { argValue = o; k = j; break; } } } //从方法调用端获得匹配参数成功,则从参数列表中移除 if (argValue != null) { args.remove(k); } } if (argValue == null) { argValue = dVs.get(i).getValue(); //设置默认值 } //如果参数值是方法,则将方法设置到 NameSpace 中 //如果参数值是变量,则将变量设置到NameSpace 中 if (argValue instanceof TshMethod) { localNameSpace.setMethod(paramName, (TshMethod) argValue); } else { localNameSpace.setLocalVariable(paramName, argValue, interpreter.getStrictJava()); } } catch (UtilEvalError e3) { throw e3.toEvalError(callerInfo, callstack); } } if (!overrideNameSpace) callstack.push(localNameSpace); // 调用方法的{...}的内容 Object ret = methodBody.eval(callstack, interpreter, true/*override*/); CallStack returnStack = callstack.copy(); if (!overrideNameSpace) callstack.pop(); ReturnControl retControl = null; //返回值较验 if (ret instanceof ReturnControl) { retControl = (ReturnControl) ret; if (Utils.eq(retControl.kind, retControl.RETURN)) ret = ((ReturnControl) ret).value; else throw new EvalError("'continue' or 'break' in method body", retControl.returnPoint, returnStack); if (returnType == Void.TYPE && ret != Primitive.VOID) throw new EvalError("Cannot return value from void method", retControl.returnPoint, returnStack); } return ret; }
方法调用参数处理这一块大家可能有点迷糊,来看几个例子
def fn(x=1,z,*args,f,n,**kwargs){ print(x,z,f,n) print(args) print(kwargs) } a = [1,3,4,5,6] m = {'n':2,'m':3} fn(*a,**m,x=3,y=2)
执行结果:
根据 python 的语法,上述*a 将 a 集合拆成 1,3,4,5,6 ,**m将m的 map 拆成n=x,m=3,因此上述方法调用变成了 fn(1,3,4,5,6,n=2,m=3,x=3,y=2),而接收参数,先对fn内方法参数排序,fn(x=1,z,f,n,*args,**kwargs),
先将 x=3匹配x,并从方法参数列表中移除,此时列表为 fn(1,3,4,5,6,n=2,m=3,y=2)
没有特定指明 z,因此 z 取第0个位置调用参数值,z=1,并将1从参数列表移除,参数列表剩fn(3,4,5,6,n=2,m=3,y=2)
同理匹配掉 z=3,n=2,参数列表剩下 fn(4,5,6,m=3,y=2),根据 python语法,*args吃掉所有没有指定参数名的调用参数,因此 args=[4,5,6],**kwargs吃掉
所有指定参数名的调用参数,m=3,y=2被,kwargs 吃掉,因此就有了上面结果,我相信理解原理后,再来了解源码,应该有所帮助。
到这里,己经将比较重要的源码己经解释清楚了,相信了解这些以后,如果需要修改源码来达到公司的要求,相信容易很多。
扩展
有了这些基本的语法以后,后面可以开发一些连接数据库的方法,即可通过脚本来操作数据库了。
同时可以集成selenium,通过脚本来操作浏览器,进行测试或爬取数据等,就是扩展的地方还有很多,感兴趣的小伙伴可以拿源码过去修改,满足自己的业务需求。