clang token def

clang在解析token的时候用到了clang::tok::TokenKind类型：

(gdb) ptype clang::tok::TokenKind
type = enum clang::tok::TokenKind {unknown, eof, eod, code_completion, comment, identifier, raw_identifier, numeric_constant, char_constant, wide_char_constant, utf8_char_constant, 
    utf16_char_constant, utf32_char_constant, string_literal, wide_string_literal, angle_string_literal, utf8_string_literal, utf16_string_literal, utf32_string_literal, l_square, 
    r_square, l_paren, r_paren, l_brace, r_brace, period, ellipsis, amp, ampamp, ampequal, star, starequal, plus, plusplus, plusequal, minus, arrow, minusminus, minusequal, tilde, 
    exclaim, exclaimequal, slash, slashequal, percent, percentequal, less, lessless, lessequal, lesslessequal, greater, greatergreater, greaterequal, greatergreaterequal, caret, 
    caretequal, pipe, pipepipe, pipeequal, question, colon, semi, equal, equalequal, comma, hash, hashhash, hashat, periodstar, arrowstar, coloncolon, at, lesslessless, 
    greatergreatergreater, kw_auto, kw_break, kw_case, kw_char, kw_const, kw_continue, kw_default, kw_do, kw_double, kw_else, kw_enum, kw_extern, kw_float, kw_for, kw_goto, kw_if, 
    kw_inline, kw_int, kw_long, kw_register, kw_restrict, kw_return, kw_short, kw_signed, kw_sizeof, kw_static, kw_struct, kw_switch, kw_typedef, kw_union, kw_unsigned, kw_void, 
    kw_volatile, kw_while, kw__Alignas, kw__Alignof, kw__Atomic, kw__Bool, kw__Complex, kw__Generic, kw__Imaginary, kw__Noreturn, kw__Static_assert, kw__Thread_local, kw___func__, 
    kw___objc_yes, kw___objc_no, kw_asm, kw_bool, kw_catch, kw_class, kw_const_cast, kw_delete, kw_dynamic_cast, kw_explicit, kw_export, kw_false, kw_friend, kw_mutable, kw_namespace, 
    kw_new, kw_operator, kw_private, kw_protected, kw_public, kw_reinterpret_cast, kw_static_cast, kw_template, kw_this, kw_throw, kw_true, kw_try, kw_typename, kw_typeid, kw_using, 
    kw_virtual, kw_wchar_t, kw_alignas, kw_alignof, kw_char16_t, kw_char32_t, kw_constexpr, kw_decltype, kw_noexcept, kw_nullptr, kw_static_assert, kw_thread_local, kw_concept, 
    kw_requires, kw_co_await, kw_co_return, kw_co_yield, kw__Decimal32, kw__Decimal64, kw__Decimal128, kw___null, kw___alignof, kw___attribute, kw___builtin_choose_expr, 
    kw___builtin_offsetof, kw___builtin_types_compatible_p, kw___builtin_va_arg, kw___extension__, kw___imag, kw___int128, kw___label__, kw___real, kw___thread, kw___FUNCTION__, 
    kw___PRETTY_FUNCTION__, kw___auto_type, kw_typeof, kw___FUNCDNAME__, kw___FUNCSIG__, kw_L__FUNCTION__, kw___is_interface_class, kw___is_sealed, kw___is_destructible, 
    kw___is_nothrow_destructible, kw___is_nothrow_assignable, kw___is_constructible, kw___is_nothrow_constructible, kw___has_nothrow_assign, kw___has_nothrow_move_assign, 
    kw___has_nothrow_copy, kw___has_nothrow_constructor, kw___has_trivial_assign, kw___has_trivial_move_assign, kw___has_trivial_copy, kw___has_trivial_constructor, 
    kw___has_trivial_move_constructor, kw___has_trivial_destructor, kw___has_virtual_destructor, kw___is_abstract, kw___is_base_of, kw___is_class, kw___is_convertible_to, kw___is_empty, 
    kw___is_enum, kw___is_final, kw___is_literal, kw___is_pod, kw___is_polymorphic, kw___is_trivial, kw___is_union, kw___is_trivially_constructible, kw___is_trivially_copyable, 
    kw___is_trivially_assignable, kw___underlying_type, kw___is_lvalue_expr, kw___is_rvalue_expr, kw___is_arithmetic, kw___is_floating_point, kw___is_integral, kw___is_complete_type, 
    kw___is_void, kw___is_array, kw___is_function, kw___is_reference, kw___is_lvalue_reference, kw___is_rvalue_reference, kw___is_fundamental, kw___is_object, kw___is_scalar, 
    kw___is_compound, kw___is_pointer, kw___is_member_object_pointer, kw___is_member_function_pointer, kw___is_member_pointer, kw___is_const, kw___is_volatile, kw___is_standard_layout, 
    kw___is_signed, kw___is_unsigned, kw___is_same, kw___is_convertible, kw___array_rank, kw___array_extent, kw___private_extern__, kw___module_private__, kw___declspec, kw___cdecl, 
    kw___stdcall, kw___fastcall, kw___thiscall, kw___vectorcall, kw___forceinline, kw___unaligned, kw___super, kw___global, kw___local, kw___constant, kw___private, kw___generic, 
    kw___kernel, kw___read_only, kw___write_only, kw___read_write, kw___builtin_astype, kw_vec_step, kw___builtin_omp_required_simd_align, kw_pipe, kw___pascal, kw___vector, kw___pixel, 
    kw___bool, kw_half, kw___bridge, kw___bridge_transfer, kw___bridge_retained, kw___bridge_retain, kw___covariant, kw___contravariant, kw___kindof, kw__Nonnull, kw__Nullable, 
    kw__Null_unspecified, kw___ptr64, kw___ptr32, kw___sptr, kw___uptr, kw___w64, kw___uuidof, kw___try, kw___finally, kw___leave, kw___int64, kw___if_exists, kw___if_not_exists, 
    kw___single_inheritance, kw___multiple_inheritance, kw___virtual_inheritance, kw___interface, kw___builtin_convertvector, kw___unknown_anytype, annot_cxxscope, annot_typename, 
    annot_template_id, annot_primary_expr, annot_decltype, annot_pragma_unused, annot_pragma_vis, annot_pragma_pack, annot_pragma_parser_crash, annot_pragma_captured, annot_pragma_dump, 
    annot_pragma_msstruct, annot_pragma_align, annot_pragma_weak, annot_pragma_weakalias, annot_pragma_redefine_extname, annot_pragma_fp_contract, annot_pragma_ms_pointers_to_members, 
    annot_pragma_ms_vtordisp, annot_pragma_ms_pragma, annot_pragma_opencl_extension, annot_pragma_openmp, annot_pragma_openmp_end, annot_pragma_loop_hint, annot_module_include, 
    annot_module_begin, annot_module_end, NUM_TOKENS}

这个类型是间接定义得到的，首先定义了clang/Basic/TokenKinds.def文件，里面定义了各种TokenKind的类型，然后在TokenKinds.h中定义各种enum，通过实现不同的宏实现不同类型的定义：

namespace clang {


namespace tok {


/// \brief Provides a simple uniform namespace for tokens from all C languages.
enum TokenKind : unsigned short {
#define TOK(X) X,
#include "clang/Basic/TokenKinds.def"
  NUM_TOKENS
};

}

}