2021-07-05 ycdxsb tools10 分钟读完 (大约 1551 个字) 0次访问

C/C++文件解析

由于主要研究C和C++，对函数、变量、类型提取等还是会经常用的，但是直接抱着ANTLR4手撸Visitor还是太傻了，所以调研了一下已有的一些工具，康康有没有好的解决方案

主要介绍三个工具，分别是lizard、pycparser、libclang

lizard#

lizard是一个代码分析的python包，语言包括但不限于c/cpp、

java，同时除了分析源代码文件，它也有一些统计代码量、检测抄袭啥的功能。当然我只关心它的源代码分析部分，它有个比较好的优点就是对源代码的分析不依赖于头文件，所以我们不用管它的头文件是不是缺失的问题。

代码量统计#

通过命令行即可调用

$ lizard openssl/ssl/bio_ssl.c 
================================================
  NLOC    CCN   token  PARAM  length  location  
------------------------------------------------
       4      1     10      1       4 BIO_f_ssl@51-54@openssl/ssl/bio_ssl.c
      12      2     63      1      15 ssl_new@56-70@openssl/ssl/bio_ssl.c
      17      5     88      1      19 ssl_free@72-90@openssl/ssl/bio_ssl.c
      61     16    294      4      69 ssl_read@92-160@openssl/ssl/bio_ssl.c
      54     14    273      4      61 ssl_write@162-222@openssl/ssl/bio_ssl.c
     157     43    863      4     169 ssl_ctrl@224-392@openssl/ssl/bio_ssl.c
      17      2     76      3      18 ssl_callback_ctrl@394-411@openssl/ssl/bio_ssl.c
       7      1     39      2       8 ssl_puts@413-420@openssl/ssl/bio_ssl.c
      15      4     94      1      18 BIO_new_buffer_ssl_connect@422-439@openssl/ssl/bio_ssl.c
      14      4     91      1      17 BIO_new_ssl_connect@441-457@openssl/ssl/bio_ssl.c
      17      4     87      2      19 BIO_new_ssl@459-477@openssl/ssl/bio_ssl.c
      15      6    111      2      15 BIO_ssl_copy_session_id@479-493@openssl/ssl/bio_ssl.c
      11      5     65      1      12 BIO_ssl_shutdown@495-506@openssl/ssl/bio_ssl.c
1 file analyzed.
==============================================================
NLOC    Avg.NLOC  AvgCCN  Avg.token  function_cnt    file
--------------------------------------------------------------
    438      30.8     8.2      165.7        13     openssl/ssl/bio_ssl.c

===========================================================================================================
!!!! Warnings (length > 1000 or cyclomatic_complexity > 15 or nloc > 1000000 or parameter_count > 100) !!!!
================================================
  NLOC    CCN   token  PARAM  length  location  
------------------------------------------------
      61     16    294      4      69 ssl_read@92-160@openssl/ssl/bio_ssl.c
     157     43    863      4     169 ssl_ctrl@224-392@openssl/ssl/bio_ssl.c
==========================================================================================
Total nloc   Avg.NLOC  AvgCCN  Avg.token   Fun Cnt  Warning cnt   Fun Rt   nloc Rt
------------------------------------------------------------------------------------------
       438      30.8     8.2      165.7       13            2      0.15    0.54

函数相关#

>>> import lizard
>>> p = lizard.analyze_file("openssl/ssl/bio_ssl.c")
>>> p.__dict__
{'function_list': [<lizard.FunctionInfo object at 0x7f13f2e5ecf8>, <lizard.FunctionInfo object at 0x7f13f2e5eb00>, <lizard.FunctionInfo object at 0x7f13f2e5eeb8>, <lizard.FunctionInfo object at 0x7f13f2e5e978>, <lizard.FunctionInfo object at 0x7f13f2e5ebe0>, <lizard.FunctionInfo object at 0x7f13f2e5e9e8>, <lizard.FunctionInfo object at 0x7f13f2e5ea20>, <lizard.FunctionInfo object at 0x7f13f2e532e8>, <lizard.FunctionInfo object at 0x7f13f2e690f0>, <lizard.FunctionInfo object at 0x7f13f2e69198>, <lizard.FunctionInfo object at 0x7f13f2e69128>, <lizard.FunctionInfo object at 0x7f13f2e691d0>, <lizard.FunctionInfo object at 0x7f13f2e69160>], 'filename': 'openssl/ssl/bio_ssl.c', 'nloc': 438, 'token_count': 2364}

可以看到主要是保留了函数的信息和一些基本信息，接着看看函数信息

1
2
3

>>> f = p.function_list[0]
>>> dir(f)
['__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', 'add_parameter', 'add_to_function_name', 'add_to_long_name', 'cyclomatic_complexity', 'end_line', 'fan_in', 'fan_out', 'filename', 'full_parameters', 'general_fan_out', 'length', 'location', 'long_name', 'name', 'name_in_space', 'nloc', 'parameter_count', 'parameters', 'start_line', 'token_count', 'top_nesting_level', 'unqualified_name']

主要还是一些很基本的信息，示例函数如下

static int ssl_new(BIO *bi)
{
    BIO_SSL *bs = OPENSSL_zalloc(sizeof(*bs));

    if (bs == NULL) {
        ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE);
        return 0;
    }
    BIO_set_init(bi, 0);
    BIO_set_data(bi, bs);
    /* Clear all flags */
    BIO_clear_flags(bi, ~0);

    return 1;
}

>>> f.start_line
56
>>> f.end_line
70
>>> f.location
' ssl_new@56-70@openssl/ssl/bio_ssl.c'
>>> f.name
'ssl_new'
>>> f.long_name
'ssl_new( BIO * bi)'
>>> f.parameters
['bi']
>>> f.full_parameters
['BIO * bi']

对于函数调用等的分析是没有的，这个作者也在README中说了。

pycparser#

是一个用python解析C语言的库，可定制性比lizard更高，目前主要兼容C99标准，它内置c_ast用于访问AST树，缺点是只能对一些标准的头文件进行忽略，当缺失头文件时，由于AST解析失败，会出错，需要自己更正直到产生正确的中间文件。具体可参考(https://eli.thegreenplace.net/2015/on-parsing-c-type-declarations-and-fake-headers)，值得一提的是，即使正确的处理得到了中间文件，parser依然可能会处理失败。

这里我们都使用base64的代码进行体验，里面有for循环也有if语句等。

pycparser主要在AST树上做事，所以写代码的时候可以多参照c_ast.py和它的examples。

访问节点示例1#

示例1主要访问函数定义、参数定义、变量定义等。

from __future__ import print_function
import sys

from pycparser import c_parser, c_ast, parse_file
from pycparser import  c_generator


def extract_functions(ast):
    class DeclVisitor(c_ast.NodeVisitor):
        def visit_Decl(self,node):
            global variables
            variables.append(generator.visit(node))

    class ParamListVisitor(c_ast.NodeVisitor):
        def visit_ParamList(self,node):
            global params
            params.append(generator.visit(node))

    class FuncDefVisitor(c_ast.NodeVisitor):
        def visit_FuncDef(self,node):
            functions[node.decl.name] = {}
            functions[node.decl.name]['position'] = "%s" % node.decl.coord
            param_v = ParamListVisitor()
            global params
            params = []
            param_v.visit(node)
            functions[node.decl.name]['params'] = params

            variables_v = DeclVisitor()
            global variables
            variables = []
            variables_v.visit(node.body)
            functions[node.decl.name]['variables'] = variables
    v = FuncDefVisitor()
    v.visit(ast)
    return functions

if __name__ == "__main__":
    if len(sys.argv) > 1:
        filename  = sys.argv[1]
    else:
        exit(0)
    functions = {}
    params = []
    variables = []
    generator = c_generator.CGenerator()
    ast = parse_file(filename,use_cpp=True,cpp_path='gcc',cpp_args=['-E','-I/root/source_code_process/cparser/utils/fake_libc_include'])
    functions = extract_functions(ast)
    print(functions)
'''
{'base64_encode': {'variables': ['int s', 'unsigned int i', 'unsigned int j', 'unsigned char c', 'unsigned char l'], 'params': ['const unsigned char *in, unsigned int inlen, char *out'], 'position': 'base64/base64.c:73:1'}, 'base64_decode': {'variables': ['unsigned int i', 'unsigned int j', 'unsigned char c'], 'params': ['const char *in, unsigned int inlen, unsigned char *out'], 'position': 'base64/base64.c:122:1'}}
'''

访问节点示例2#

示例2主要想知道函数调用了哪些函数，也就是想生成函数调用图

from __future__ import print_function
import sys

from pycparser import c_parser, c_ast, parse_file
from pycparser import  c_generator


def extract_functions(ast):
    class FuncCallVisitor(c_ast.NodeVisitor):
        def visit_FuncCall(self,node):
            global funccalls
            funccalls.append(node.name.name)

    class FuncDefVisitor(c_ast.NodeVisitor):
        def visit_FuncDef(self,node):
            functions[node.decl.name] = {}
            functions[node.decl.name]['position'] = "%s" % node.decl.coord
            func_v = FuncCallVisitor()
            global funccalls
            funccalls = []
            func_v.visit(node)
            functions[node.decl.name]['funccalls'] = funccalls

    v = FuncDefVisitor()
    v.visit(ast)
    return functions



if __name__ == "__main__":
    if len(sys.argv) > 1:
        filename  = sys.argv[1]
    else:
        exit(0)
    functions = {}
    funccalls = []
    generator = c_generator.CGenerator()
    ast = parse_file(filename,use_cpp=True,cpp_path='gcc',cpp_args=['-E','-I/root/source_code_process/cparser/utils/fake_libc_include'])
    functions = extract_functions(ast)
    print(functions)
'''
{'main': {'position': 'base64/main.c:30:1', 'funccalls': ['test', 'test', 'test', 'test', 'test', 'test', 'test']}, 'test': {'position': 'base64/main.c:9:1', 'funccalls': ['malloc', 'malloc', 'assert', 'assert', 'assert', 'assert', 'assert', 'assert', 'free', 'free']}}
'''

libclang#

libclang需要先装llvm，从一个纯净ubuntu的docker开始，命令如下

apt install vim gcc g++ cmake python2 python3 python3-distutils python3-pip
tar -xvf llvm-11.0.0.src.tar.xz
mv llvm-11.0.0.src llvm
tar -xvf clang-11.0.0.src.tar.xz
mv clang-11.0.0.src clang
mv clang llvm/tools/
tar -xvf compiler-rt-11.0.0.src.tar.xz
mv compiler-rt-11.0.0.src llvm/projects/compiler-rt
tar -xvf libcxx-11.0.0.src.tar.xz
mv libcxx-11.0.0.src llvm/projects/libcxx
tar -xvf libcxxabi-11.0.0.src.tar.xz
mv libcxxabi-11.0.0.src llvm/projects/libcxxabi
tar -xvf lldb-11.0.0.src.tar.xz
mv lldb-11.0.0.src llvm/tools/lldb
tar -xvf lld-11.0.0.src.tar.xz
mv lld-11.0.0.src llvm/tools/lld
tar -xvf clang-tools-extra-11.0.0.src.tar.xz
mv clang-tools-extra-11.0.0.src llvm/tools/clang/tools/extra
cd llvm
mkdir build && cd build
cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release ..
make -j4
pip3 install clang==11.0

算了，先空着吧，等用到了再回来填坑

参考资料#

#tools