抽象语法树(Abstract Syntax Tree)是源代码的抽象语法结构的树状表示。
抽象语法树是解析器(parser)的产物,解析器广义来说输入一般是程序的源码,输出一般是语法树(syntax tree,也叫parse tree等)或抽象语法树。进一步剥开来,广义的解析器里一般会有扫描器(scanner,也叫tokenizer或者lexical analyzer,词法分析器),以及狭义的解析器(parser,也叫syntax analyzer,语法分析器)。扫描器的输入一般是文本,经过词法分析,输出是将文本切割为单词的流。狭义的解析器输入是单词的流,经过语法分析,输出是语法树或者精简过的AST。
例如:将i = a + b * c作为源代码输入到解析器里,则广义上的解析器的工作流程如下图:
二、用途
erlang beam是寄存器虚拟机,因此erlang的源码会被erlang解析器的词法分析变为AST,然后经过树形遍历解析器给转换为汇编字节码。
三、抽象语法树初识
1、创建一个test1.erl的文件,输入以下代码:
-module(test1).
-export([start/0]).
-compile({parse_transform, test_parser}).
start() ->
"hello world".
2、创建一个test_parser.erl的文件,输入以下代码:
-module(test_parser).
-export([parse_transform/2]).
parse_transform(AST, _Options) ->
io:format("old:~w~n~n", [AST]),
Acc = parse_ast(AST, []),
io:format("new:~w~n~n", [Acc]),
Acc.
parse_ast([{attribute, _, _, _} = H | R], Acc) ->
parse_ast(R, [H | Acc]);
parse_ast([{function, _Line, _Fun, _Arity, _Args} = H | R], Acc) ->
parse_ast(R, [parse_fun(H) | Acc]);
parse_ast([H | R], Acc) ->
parse_ast(R, [H | Acc]);
parse_ast([], Acc) ->
lists:reverse(Acc).
parse_fun({function, Line, Fun, Arity, Clause}) ->
{function, Line, Fun, Arity, parse_clause(Clause, [])}.
parse_clause([{clause, Line, P, Guard, Return}], Acc) ->
[{clause, Line, P, Guard, parse_return(Return, [])} | Acc].
parse_return([{string, Line, Value} | R], Acc) ->
parse_return(R, [{bin, Line, [{bin_element, Line, {string, Line, Value},default,default}]} | Acc]);
parse_return([], Acc) ->
lists:reverse(Acc).
3、编译
1> c(test_parser).
{ok,test_parser}
2> c(test1).
old:[{attribute,1,file,{[116,101,115,116,49,46,101,114,108],1}},{attribute,1,module,test1},{attribute,3,export,[{start,0}]},
{function,7,start,0,[{clause,7,[],[],[{string,8,[104,101,108,108,111,32,119,111,114,108,100]}]}]},
{eof,9}]
new:[{attribute,1,file,{[116,101,115,116,49,46,101,114,108],1}},{attribute,1,module,test1},{attribute,3,export,[{start,0}]},{function,7,start,0,[{clause,7,[],[],[{bin,8,[{bin_element,8,{string,8,[104,101,108,108,111,32,119,111,114,108,100]},default,default}]}]}]},{eof,9}]
{ok,test1}
先不管io:format的打印,我们直接执行test1:start(),
3> test1:start().
<<"hello world">>
发现终端输出的是二进制!而我们源码明明写的是个字符串!!破编译器对我们的代码干了什么!!
别惊慌,这时候我们就可以来看看我们打印的抽象语法树了。
先看old,attribute都是源码文件的属性,例如file、export、import、module名等,可以略过,以及eof表示源码文件的结尾,也可以忽略。直接定位到function元组,{function, Line, Name, Arity, Clauses}表示一个函数的源码行、函数名、形参数、模式匹配列表,再定位到模式匹配列表clause,其表示了一个函数的多个模式匹配和返回值,当前start函数就一个匹配,因此clause只有一个tuple元素,返回值就是一个{string, Line, "hello world"},
那么我们是不是修改这个{string, Line, "hello world"}为一个二进制字符串,就能达到我们演示的效果呢?是的!
这就是test1.erl中-compile({parse_transform, test_parser}).的作用,erlang的解析器会在词法分析完成后,调用我们自己指定的函数对ast进行二次更改,于是test_parser.erl里面就将start函数的返回值更改为了一个二进制。然后返回一个新的AST,再经树形解析编译为erlang汇编字节码文件,供erlang vm调用。
四、应用
lager日志库 https://github.com/erlang-lager/lager
以前一直疑惑使用lager日志库为什么要在编译选项上加一个{parse_transform, lager_transform},现在也懂了,
就是编译我们项目每一个源文件时,都将最终的抽象语法树再次添加一些特性代码。
这里我们可以打印一下,现在我在一个叫erlserver_app.erl的文件里加入lager日志打印代码:
-module(erlserver_app).
-behaviour(application).
-export([start/2, stop/1]).
-compile([{parse_transform, lager_transform}]).
start(_StartType, _StartArgs) ->
lager:warning("12345678987654321~w~n", ["abcdedeba"]),
erlserver_sup:start_link().
%%--------------------------------------------------------------------
stop(_State) ->
ok.
然后在lager_transform里继续像上面那样打印,
最终编译输出:
===> old:
[{attribute,1,file,{[47,104,111,109,101,47,108,105,107,117,110,47,115,116,117,100,121,47,101,114,108,97,110,103,47,101,114,108,115,101,114,118,101,114,47,95,98,117,105,108,100,47,100,101,102,97,117,108,116,47,108,105,98,47,101,114,108,115,101,114,118,101,114,47,115,114,99,47,101,114,108,115,101,114,118,101,114,95,97,112,112,46,101,114,108],1}},{attribute,6,module,erlserver_app},{attribute,8,behaviour,application},{attribute,11,export,[{start,2},{stop,1}]},{attribute,13,compile,[]},{function,19,start,2,[{clause,19,[{var,19,'_StartType'},{var,19,'_StartArgs'}],[],[{call,20,{remote,20,{atom,20,lager},{atom,20,warning}},[{string,20,[49,50,51,52,53,54,55,56,57,56,55,54,53,52,51,50,49,126,119,126,110]},{cons,20,{string,20,[97,98,99,100,101,100,101,98,97]},{nil,20}}]},{call,21,{remote,21,{atom,21,erlserver_sup},{atom,21,start_link}},[]}]}]},{function,24,stop,1,[{clause,24,[{var,24,'_State'}],[],[{atom,25,ok}]}]},{eof,30}]
===> new:
[{attribute,1,file,{[47,104,111,109,101,47,108,105,107,117,110,47,115,116,117,100,121,47,101,114,108,97,110,103,47,101,114,108,115,101,114,118,101,114,47,95,98,117,105,108,100,47,100,101,102,97,117,108,116,47,108,105,98,47,101,114,108,115,101,114,118,101,114,47,115,114,99,47,101,114,108,115,101,114,118,101,114,95,97,112,112,46,101,114,108],1}},{attribute,6,module,erlserver_app},{attribute,6,lager_records,[]},{attribute,8,behaviour,application},{attribute,11,export,[{start,2},{stop,1}]},{attribute,13,compile,[]},{function,19,start,2,[{clause,19,[{var,19,'_StartType'},{var,19,'_StartArgs'}],[],[{'case',20,{tuple,20,[{call,20,{atom,20,whereis},[{atom,20,lager_event}]},{call,20,{atom,20,whereis},[{atom,20,lager_event}]},{call,20,{remote,20,{atom,20,lager_config},{atom,20,get}},[{tuple,20,[{atom,20,lager_event},{atom,20,loglevel}]},{tuple,20,[{integer,20,0},{nil,20}]}]}]},[{clause,20,[{tuple,20,[{atom,20,undefined},{atom,20,undefined},{var,20,'_'}]}],[],[{call,20,{'fun',20,{clauses,[{clause,20,[],[],[{tuple,20,[{atom,20,error},{atom,20,lager_not_running}]}]}]}},[]}]},{clause,20,[{tuple,20,[{atom,20,undefined},{var,20,'_'},{var,20,'_'}]}],[],[{call,20,{'fun',20,{clauses,[{clause,20,[],[],[{tuple,20,[{atom,20,error},{tuple,20,[{atom,20,sink_not_configured},{atom,20,lager_event}]}]}]}]}},[]}]},{clause,20,[{tuple,20,[{var,20,'__Piderlserver_app20'},{var,20,'_'},{tuple,20,[{var,20,'__Levelerlserver_app20'},{var,20,'__Traceserlserver_app20'}]}]}],[[{op,20,'orelse',{op,20,'/=',{op,20,'band',{var,20,'__Levelerlserver_app20'},{integer,20,16}},{integer,20,0}},{op,20,'/=',{var,20,'__Traceserlserver_app20'},{nil,20}}}]],[{call,20,{remote,20,{atom,20,lager},{atom,20,do_log}},[{atom,20,warning},{cons,20,{tuple,20,[{atom,20,application},{atom,20,erlserver}]},{cons,20,{tuple,20,[{atom,20,module},{atom,20,erlserver_app}]},{cons,20,{tuple,20,[{atom,20,function},{atom,20,start}]},{cons,20,{tuple,20,[{atom,20,line},{integer,20,20}]},{cons,20,{tuple,20,[{atom,20,pid},{call,20,{atom,20,pid_to_list},[{call,20,{atom,20,self},[]}]}]},{cons,20,{tuple,20,[{atom,20,node},{call,20,{atom,20,node},[]}]},{call,20,{remote,20,{atom,20,lager},{atom,20,md}},[]}}}}}}},{string,20,[49,50,51,52,53,54,55,56,57,56,55,54,53,52,51,50,49,126,119,126,110]},{cons,20,{string,20,[97,98,99,100,101,100,101,98,97]},{nil,20}},{integer,20,4096},{integer,20,16},{var,20,'__Levelerlserver_app20'},{var,20,'__Traceserlserver_app20'},{atom,20,lager_event},{var,20,'__Piderlserver_app20'}]}]},{clause,20,[{var,20,'_'}],[],[{atom,20,ok}]}]},{call,21,{remote,21,{atom,21,erlserver_sup},{atom,21,start_link}},[]}]}]},{function,24,stop,1,[{clause,24,[{var,24,'_State'}],[],[{atom,25,ok}]}]},{eof,30}]
看来添加的东西挺多,没事,慢慢看,我们根据官方文档
The Abstract Format一个一个对比看,
最终将添加的代码还原出来:
case {whereis(lager_event), whereis(lager_event), lager_config:get({lager_event, loglevel}, {0, []})} of
{undefined, undefined, _} ->
{error, lager_not_running};
{undefined, _, _} ->
{error, sink_not_configured, lager_event};
{__Piderlserver_app20, _, {__Levelerlserver_app20, __Traceserlserver_app20}} when __Levelerlserver_app20 band 16 /= 0 orelse __Traceserlserver_app20 /= nil ->
lager:do_log(warning, [{application, erlserver}, {atom, erlserver_app}, {function, start}, {line, 20}, {pid, pid_to_list(self())}, {node, node()}, lager:md()],
[49,50,51,52,53,54,55,56,57,56,55,54,53,52,51,50,49,126,119,126,110], [[97,98,99,100,101,100,101,98,97], []], 4096, 16, __Levelerlserver_app20,
__Traceserlserver_app20, lager_event, __Piderlserver_app20);
_ ->
ok
end
小小一句lager:warning()原来有这么多弯弯绕啊!
五、erlang汇编字节码
一直说erlang汇编字节码,但是具体是个什么东西,还没有认识。
下面新建一个文件test.erl,输入以下内容:
-module(test).
-export([start/2]).
start(A, B) ->
abs(A) + abs(B).
1、编译.S汇编码
erlc -S test.erl
2、编译.dis汇编码
erl
c(test).
erts_debug:df(test).
然后打开得到的test.S文件:
{module, test}. %% version = 0
{module, test}. %% version = 0
{exports, [{module_info,0},{module_info,1},{start,2}]}.
{attributes, []}.
{labels, 7}.
{function, start, 2, 2}.
{label,1}.
{line,[{location,"test.erl",4}]}.
{func_info,{atom,test},{atom,start},2}.
{label,2}.
{line,[{location,"test.erl",5}]}.
{gc_bif,abs,{f,0},2,[{x,0}],{x,0}}.
{line,[{location,"test.erl",5}]}.
{gc_bif,abs,{f,0},2,[{x,1}],{x,1}}.
{line,[{location,"test.erl",5}]}.
{gc_bif,'+',{f,0},2,[{x,0},{x,1}],{x,0}}.
return.
{function, module_info, 0, 4}.
{label,3}.
{line,[]}.
{func_info,{atom,test},{atom,module_info},0}.
{label,4}.
{move,{atom,test},{x,0}}.
{line,[]}.
{call_ext_only,1,{extfunc,erlang,get_module_info,1}}.
{function, module_info, 1, 6}.
{label,5}.
{line,[]}.
{func_info,{atom,test},{atom,module_info},1}.
{label,6}.
{move,{x,0},{x,1}}.
{move,{atom,test},{x,0}}.
{line,[]}.
{call_ext_only,2,{extfunc,erlang,get_module_info,2}}.
每一个function元组标明了模块中的函数,我们直接定位到function start模块,每个label表示pc指针的执行顺序,跳过label1,我们直接看label2,有许多寄存器x0,x1,...xn y0,y1,...yn。x寄存器是用来存储函数入参的,特殊的x0寄存器,函数第一个入参是存放在里面,并且也用来作为函数返回值,而y寄存器是栈上分配的临时寄存器,
有了以上知识,我们直接看汇编码:
(忽略某些无用行)
{gc_bif,abs,{f,0},2,[{x,0}],{x,0}}. //用x0去调用bif函数abs,存储结果到x0寄存器,
{gc_bif,abs,{f,0},2,[{x,1}],{x,1}}. //用x1去调用bif函数abs,存储结果到x1寄存器,
{gc_bif,'+',{f,0},2,[{x,0},{x,1}],{x,0}}. //执行bif函数+, x0+x1,结果放入x0寄存器
return. //返回,直接返回x0的值
这就是.S汇编字节码,它具有一定可读性,方便人和机器阅读再打开.dis汇编码文件:
00007FDAE5183770: i_func_info_IaaI 0 test start 2
00007FDAE5183798: i_gc_bif1_jIsId j(0000000000000000) abs/1 x(0) 2 x(0)
00007FDAE51837C8: i_gc_bif1_jIsId j(0000000000000000) abs/1 x(1) 2 x(1)
00007FDAE51837F8: i_plus_jIxxd j(0000000000000000) 2 x(0) x(1) x(0)
00007FDAE5183828: return
00007FDAE5183830: i_func_info_IaaI 0 test module_info 0
00007FDAE5183858: move_cr test r(0)
00007FDAE5183868: allocate_tt 0 1
00007FDAE5183878: call_bif_e erlang:get_module_info/1
00007FDAE5183888: deallocate_return_Q 0
00007FDAE5183898: i_func_info_IaaI 0 test module_info 1
00007FDAE51838C0: move_rx r(0) x(1)
00007FDAE51838D0: move_cr test r(0)
00007FDAE51838E0: allocate_tt 0 2
00007FDAE51838F0: call_bif_e erlang:get_module_info/2
00007FDAE5183900: deallocate_return_Q 0
其实大体上差不多,不过多了真是调用的函数和执行地址,但是erlang的汇编操作码太多,因此这个文件可读性很差,如果想搞懂具体指令作用可以去查阅相关文档。