strtok in "string.h"

/*
 * Copyright (c) 1988 Regents of the University of California.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 * This product includes software developed by the University of
 * California, Berkeley and its contributors.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include <string.h>

char *
_DEFUN (strtok_r, (s, delim, lasts),
 register char *s _AND
 register const char *delim _AND
 char **lasts)
{
 register char *spanp;
 register int c, sc;
 char *tok;

 if (s == NULL && (s = *lasts) == NULL)
  return (NULL);

 /*
  * Skip (span) leading delimiters (s += strspn(s, delim), sort of).
  */
cont:
 c = *s++;
 for (spanp = (char *)delim; (sc = *spanp++) != 0;) {
  if (c == sc)
   goto cont;
 }

 if (c == 0) {  /* no non-delimiter characters */
  *lasts = NULL;
  return (NULL);
 }
 tok = s - 1;

 /*
  * Scan token (scan for delimiters: s += strcspn(s, delim), sort of).
  * Note that delim must have one NUL; we stop if we see that, too.
  */
 for (;;) {
  c = *s++;
  spanp = (char *)delim;
  do {
   if ((sc = *spanp++) == c) {
    if (c == 0)
     s = NULL;
    else
     s[-1] = 0;
    *lasts = s;
    return (tok);
   }
  } while (sc != 0);
 }
 /* NOTREACHED */
}

以上是libc/string/strtok_r.c得源码
实际上strtok的原码是

#undef  __STRICT_ANSI__
#include <string.h>
#include <_ansi.h>
#include <reent.h>

#ifndef _REENT_ONLY

char *
_DEFUN (strtok, (s, delim),
 register char *s _AND
 register const char *delim)
{
 return strtok_r (s, delim, &(_REENT->_new._reent._strtok_last));
}
#endif

还是调用的上面这个strtok_r

某个微软的具体实现

char * __cdecl strtok (
        char * string,
        const char * control
        )
{
        unsigned char *str;
        const unsigned char *ctrl = control;

        unsigned char map[32];
        int count;

#ifdef _MT
        _ptiddata ptd = _getptd();
#else  /* _MT */
        static char *nextoken;
#endif  /* _MT */

        /* Clear control map */
        for (count = 0; count < 32; count++)
                map[count] = 0;

        /* Set bits in delimiter table */
        do {
                map[*ctrl >> 3] |= (1 << (*ctrl & 7));
        } while (*ctrl++);

        /* Initialize str. If string is NULL, set str to the saved
         * pointer (i.e., continue breaking tokens out of the string
         * from the last strtok call) */
        if (string)
                str = string;
        else
#ifdef _MT
                str = ptd->_token;
#else  /* _MT */
                str = nextoken;
#endif  /* _MT */

        /* Find beginning of token (skip over leading delimiters). Note that
         * there is no token iff this loop sets str to point to the terminal
         * null (*str == '/0') */
        while ( (map[*str >> 3] & (1 << (*str & 7))) && *str )
                str++;

        string = str;

        /* Find the end of the token. If it is not the end of the string,
         * put a null there. */
        for ( ; *str ; str++ )
                if ( map[*str >> 3] & (1 << (*str & 7)) ) {
                        *str++ = '/0';
                        break;
                }

        /* Update nextoken (or the corresponding field in the per-thread data
         * structure */
#ifdef _MT
        ptd->_token = str;
#else  /* _MT */
        nextoken = str;
#endif  /* _MT */

        /* Determine if a token has been found. */
        if ( string == str )
                return NULL;
        else
                return string;
}

在strtok中，第二个参数是分隔符列表，假如是"abcd"，那么，'a','b','c','d',"ab","bc",...,"abc",..."abcd"等，都将被作为词法分隔单元！