java.util.StringTokenizer源码学习记录

2019独角兽企业重金招聘Python工程师标准>>>

StringTokenizer stringTokenizer = new StringTokenizer("1aksdhka111as23sda", "as");
while (stringTokenizer.hasMoreTokens()) {
    System.out.println(stringTokenizer.nextToken());
}

public StringTokenizer(String str, String delim, boolean returnDelims) {
    currentPosition = 0;
    newPosition = -1;
    delimsChanged = false;
    this.str = str; // 被处理的字符串
    maxPosition = str.length(); 
    delimiters = delim;       // 分隔符
    retDelims = returnDelims; // 是否返回分隔符
    setMaxDelimCodePoint();
}

private void setMaxDelimCodePoint() {
    if (delimiters == null) {
        maxDelimCodePoint = 0;
        return;
    }

    int m = 0;
    int c;
    int count = 0;
    for (int i = 0; i < delimiters.length(); i += Character.charCount(c)) {
        c = delimiters.charAt(i);
        // 判断字符是不是highSurrogate   就是占两个字符的一个文字
        if (c >= Character.MIN_HIGH_SURROGATE && c <= Character.MAX_LOW_SURROGATE) {
            c = delimiters.codePointAt(i);
            hasSurrogates = true; // 表示分隔符中含有Surrogate的文字  在判断是否含有分隔符时使用
        }
        if (m < c) // 分隔符delimiters中数字最大的codePoint
            m = c;
        count++;
    }
    maxDelimCodePoint = m;

    if (hasSurrogates) {
        delimiterCodePoints = new int[count]; // 保存分隔符的codePoint数组
        for (int i = 0, j = 0; i < count; i++, j += Character.charCount(c)) {
            c = delimiters.codePointAt(j);
            delimiterCodePoints[i] = c;
        }
    }
}

public boolean hasMoreTokens() {
    newPosition = skipDelimiters(currentPosition);
    return (newPosition < maxPosition);
}

private int skipDelimiters(int startPos) {
    if (delimiters == null)
        throw new NullPointerException();

    int position = startPos;
    // 这个循环的作用就是将当前指针移动到非分隔符文字的位置上
    while (!retDelims && position < maxPosition) {
        if (!hasSurrogates) { // 根据分隔符是否含有Surrogates文字 进行作用一样的不同判断
            char c = str.charAt(position);
            if ((c > maxDelimCodePoint) || (delimiters.indexOf(c) < 0))
                break;
            position++;
        } else {
            int c = str.codePointAt(position);
            if ((c > maxDelimCodePoint) || !isDelimiter(c)) {
                break;
            }
            position += Character.charCount(c);
        }
    }
    return position;
}

public String nextToken() {
    
    // 这一步是为String nextToken(String delim)方法服务的  分隔过程中改变分隔符
    currentPosition = (newPosition >= 0 && !delimsChanged) ?
        newPosition : skipDelimiters(currentPosition);

    /* Reset these anyway */
    delimsChanged = false;
    newPosition = -1;

    if (currentPosition >= maxPosition)
        throw new NoSuchElementException();
    int start = currentPosition;
    currentPosition = scanToken(currentPosition);
    return str.substring(start, currentPosition);
}

private int scanToken(int startPos) {
    int position = startPos;
    // 这个循环的作用就是将指针移动到分隔符所在的位置
    while (position < maxPosition) {
        if (!hasSurrogates) {
            char c = str.charAt(position);
            if ((c <= maxDelimCodePoint) && (delimiters.indexOf(c) >= 0))
                break;
            position++;
        } else {
            int c = str.codePointAt(position);
            if ((c <= maxDelimCodePoint) && isDelimiter(c))
                break;
            position += Character.charCount(c);
        }
    }
    // startPos==position就是startPos指向的就是分隔符 
    // 那么这次scanToken就是扫描出分隔符开始和结束位置  说明分隔符可以是多个字符组成
    if (retDelims && (startPos == position)) {
        if (!hasSurrogates) {
            char c = str.charAt(position);
            if ((c <= maxDelimCodePoint) && (delimiters.indexOf(c) >= 0))
                position++;
        } else {
            int c = str.codePointAt(position);
            if ((c <= maxDelimCodePoint) && isDelimiter(c))
                position += Character.charCount(c);
        }
    }
    return position;
}

 

转载于:https://my.oschina.net/zhuqianli/blog/1606121