LLVM clang_tokenize 的示例

最新推荐文章于 2024-10-18 16:08:35 发布

Eloudy

最新推荐文章于 2024-10-18 16:08:35 发布

阅读量366

点赞数 9

文章标签：编译器深度学习

本文链接：https://blog.csdn.net/eloudy/article/details/136453903

版权

本文介绍了如何通过Git克隆LLVM项目，配置编译环境，创建Makefile并提供了一个C++示例代码，展示了如何使用Clang库进行C++源代码分析。

摘要由CSDN通过智能技术生成

1，环境搭建

如同上篇

git clong llvm-project

git checkout llmvorg-3.4.0

# or llvmorg-18.1.rc

/bin/bash ./confiure --prefix=.......

# or cmake =Debug

make -j32

make -j install

2，示例代码

Makefile：如同上篇

LLVM_CONFIG ?= llvm-config
#CXX := clang++
ifndef VERBOSE
QUIET :=@
endif

SRC_DIR ?= $(PWD)
LDFLAGS += $(shell $(LLVM_CONFIG) --ldflags) 
COMMON_FLAGS = -Wall -Wextra
CXXFLAGS += $(COMMON_FLAGS) $(shell $(LLVM_CONFIG) --cxxflags)
LCXX :=$(shell $(LLVM_CONFIG) --cxxflags)
CPPFLAGS += $(shell $(LLVM_CONFIG) --cppflags) -I$(SRC_DIR)

CLANGLIBS = \
  -Wl,--start-group \
  -lclang \
  -lclangFrontend \
  -lclangDriver \
  -lclangSerialization \
  -lclangParse \
  -lclangSema \
  -lclangAnalysis \
  -lclangEdit \
  -lclangAST \
  -lclangLex \
  -lclangBasic \
  -Wl,--end-group

LLVMLIBS = $(shell $(LLVM_CONFIG) --libs)

PROJECT = token_analy
PROJECT_OBJECTS = token_analy.o

default: $(PROJECT)

%.o : $(SRC_DIR)/%.cpp
	@echo Compiling $*.cpp
	$(QUIET)$(CXX) -c $(CPFLAGS) $(CXXFLAGS) $<

$(PROJECT) : $(PROJECT_OBJECTS) 
	@echo Linking $@
	$(QUIET)$(CXX) -o $@ $(LDFLAGS) $^ $(CLANGLIBS) $(LLVMLIBS) -lncurses

.PHONY: clean
clean:
	$(QUIET)rm -f $(PROJECT) $(PROJECT_OBJECTS)


.PHONY: echo
echo:
	@echo "CXX 	is	$(CXX)"
	@echo "LDFLAGS 	is	$(LDFLAGS)}"
	@echo "CXXFLAGS	is	$(CXXFLAGS)"
	@echo "CPPFLAGS	is	$(CPPFLAGS)"
	@echo "SRC_DIR	is	$(SRC_DIR)"

token_analy.c:

extern "C"{
#include "clang-c/Index.h"
}

#include "llvm/Support/CommandLine.h"
#include <iostream>

using namespace llvm;

static cl::opt<std::string> FileName(cl::Positional, cl::desc("Input file"), cl::Required);

int main(int argc, char** argv)
{
	cl::ParseCommandLineOptions(argc, argv, "My tokenizer\n");
	CXIndex index = clang_createIndex(0, 0);
	const char *args[]={
		"-I/usr/include",
		"-I."
	};

	CXTranslationUnit translationUnit = clang_parseTranslationUnit(index, FileName.c_str(), args, 2, NULL, 0, CXTranslationUnit_None);
	CXFile file = clang_getFile(translationUnit, FileName.c_str());
	CXSourceLocation loc_start = clang_getLocationForOffset(translationUnit, file, 0);
	CXSourceLocation loc_end   = clang_getLocationForOffset(translationUnit, file, 60);
	CXSourceRange range = clang_getRange(loc_start, loc_end);
	unsigned numTokens = 0;
	CXToken *tokens = NULL;
	clang_tokenize(translationUnit, range, &tokens, &numTokens);

	for(unsigned i=0; i<numTokens; ++i){
		enum CXTokenKind kind = clang_getTokenKind(tokens[i]);
		CXString name = clang_getTokenSpelling(translationUnit, tokens[i]);
		switch(kind){
			case CXToken_Punctuation:
				std::cout<<"PUNCTUATION("<<clang_getCString(name)<<") ";
				break;
			case CXToken_Keyword:
				std::cout<<"KEYWORD("<<clang_getCString(name)<<") ";
				break;
			case CXToken_Identifier:
				std::cout<<"IDENTIFIER("<<clang_getCString(name)<<") ";
				break;
			case CXToken_Literal:
				std::cout<<"COMMENT("<<clang_getCString(name)<<") ";
				break;
			default:
				std::cout<<"UNKNOWN("<<clang_getCString(name)<<") ";
				break;
		
		}
		clang_disposeString(name);
	}
	std::cout<< std::endl;
	clang_disposeTokens(translationUnit, tokens, numTokens);
	clang_disposeTranslationUnit(translationUnit);
	return 0;

}

make