Tokenizer类的框架

作用：用于代码编辑器中的括号匹配测试

package com.tokenizer.cc;

import java.io.IOException;
import java.io.PushbackReader;
import java.io.Reader;
/**
 * Tokenizer class.
 * @author SunnyBoy
 * @version Time：2017年8月3日 下午3:57:00
 */
public class Tokenizer {
    private PushbackReader in;// The input Stream
    private char ch;//Current character
    private int currentLine;//Current line
    private int errors;//Number of errors seen

    public static final int SLASH_SLAH = 0;
    public static final int SLASH_STAR = 1;

    public Tokenizer( Reader inStream) {
        errors = 0;
        ch = '\0';
        currentLine = 1;
        in = new PushbackReader(inStream);
    }

    public int getLineNumber() {
        return currentLine;
    }

    public int getErrorCount() {
        return errors;
    }
    /**
     * Get the next opening or closing symbol.
     * Return false if end of file.
     * Skip past comments and character and string constants
     */
    public char getNextOpenClose() {
        while(nextChar()) {
            if(ch == '/')
                processSlash();
            else if(ch == '\''||ch =='"')
                skipQuote(ch);
            else if(ch =='(' || ch == '[' || ch == '{' || ch ==')' || ch == ']' || ch == '}')
                return ch;
        }
        return '\0';
    }
    /**
     * @return next identifier.skipping comments
     * string constants.and character constants.
     * Place identifier in currentIdNode.word and return false
     * only if end of stream is reached.
     */
    public String getNextID() {
        while(nextChar()) {
            if(ch =='/')
                processSlash();
            else if(ch =='\\')
                nextChar();
            else if(ch =='\'' || ch =='"')
                skipQuote(ch);
            else if (!Character.isDigit(ch)&&isIdChar(ch))
                return getRemainingString();
        }
        return null;
    }
    /**
     * nextChar sets ch based on the next character in the input stream.
     * putBackChar puts thecharacter back onto the stream.
     * It should be used only once after a call to nextChar.
     * Both routines adjust currentLine if necessary.
     * @return
     */
    private boolean nextChar() {
        try {
            int readVal = in.read();
            if(readVal == -1)
                return false;
            ch = (char) readVal;
            if(ch == '\n')
                currentLine++;
            return true;
        }
        catch(IOException e) {
            return false;
        }
    }

    private void putBackChar() {
        if(ch == '\n')
            currentLine--;
        try {
            in.unread((int)ch);
        }catch(IOException e) {

        }
    }

    /**
     * Precondition: We are about to process a comment;
     *              have already seen comment-start token
     * Postcondition: Stream will be set immediately after 
     *              commment-ending token
     * @param start
     */
    private void skipComment(int start) {
        if(start == SLASH_SLAH) {
            while(nextChar()&&(ch!='\n'));
            return;
        }
        boolean state = false;

        while(nextChar()) {
            if(state&&ch=='/')
                return;
            state = (ch =='*');
        }
        errors++;
        System.out.println("Unterminated comment!");
    }
    /**
     * Precondition: We are about to process a quote;
     *                  have already seen beginning quote.
     * Postcondition: Stream will be set immediately after
     *                  matching quote
     * @param quoteType
     */
    private void skipQuote(char quoteType) {
        while(nextChar()) {
            if(ch == quoteType)
                return;
            if(ch == '\n') {
                errors++;
                System.out.println("Missing closed quote at line "+ currentLine);
                return;
            }
            else if(ch == '\\')
                nextChar();
        }
    }
    /**
     * After the opening slash is seen deal with next character.
     * If it is a comment starter. process it; otherwise putback
     * the next character if it is not a newline.
     */
    private void processSlash() {
        if (nextChar()) {
            if (ch == '*') {
                // Javadoc comment
                if (nextChar() && ch != '*')
                    putBackChar();
                skipComment(SLASH_STAR);

            } else if (ch == '/')
                skipComment(SLASH_SLAH);
            else if (ch == '\n')
                putBackChar();
        }
    }
    /**
     * Return true if ch can be part of a Java identifier
     * @param ch
     * @return
     */
    private static final boolean isIdChar(char ch) {
        return Character.isJavaIdentifierPart(ch);
    }
    /**
     * Return an identifier read from input stream
     * First character is already read into ch
     * @return
     */
    private String getRemainingString() {
        String result = ""+ch;
        for(;nextChar();result+=ch)
            if(!isIdChar(ch)) {
                putBackChar();
                break;
            }
        return result;
    }
}