pyspider_analysis_4

pyspider.run.py中辅助函数解析

• read_config

  • 具体实现

    def read_config(ctx, param, value):
        if not value:
            return {}
        import json
    
        def underline_dict(d):
            if not isinstance(d, dict):
                return d
            return dict((k.replace('-', '_'), underline_dict(v)) for k, v in six.iteritems(d))
    
        config = underline_dict(json.load(value))
        ctx.default_map = config
        return config
    

  
  

  • ctx为消息传递的媒介，通常在各个command内使用

  • parm的type是click.core.Option object，具体内容可使用parm.__dict__查看，举例如下

    def test_call(ctx, parm, value):
        print(parm.__dict__)
        print("callback:", value)
        return value
    
    @cli.command()
    @click.option("-t", "--test", envvar="Path", help="Test", callback=test_call)
    @click.pass_context
    def test(ctx, **kwargs):
        print("obj:", ctx.obj)
        click.echo(kwargs)
    
    
    if __name__ == "__main__":
        test()
    
    

    parm内容为

    {   'name': 'test', 
        'opts': ['-t', '--test'], 
        'secondary_opts': [], 
        'type': STRING, 
        'required': False, 
        'callback': <function test_call at 0x000002228FCDE510>, 'nargs': 1, 
        'multiple': False, 
        'expose_value': True, 
        'default': None, 
        'is_eager': False, 
        'metavar': None, 
        'envvar': 'Path', 
        'autocompletion': None, 
        'prompt': None, 
        'confirmation_prompt': False, 
        'hide_input': False, 
        'hidden': False, '
       	'is_flag': False, 
        'flag_value': True, 
        'is_bool_flag': False, 
        'count': False, 
        'allow_from_autoenv': True, 
        'help': 'Test', 
        'show_default': False, 
        'show_choices': True, 
        'show_envvar': False}
    

  • value为命令行中[OPTIONS] [ARGS]结构中的ARGS

  • 在接下来的几个辅助函数中参数同上

  • 如果value存在，读取value所代表的json文件，交由underline_dict处理，处理过程举例如下

    def underline_dict(d):
        if not isinstance(d, dict):
            return d
        return dict((k.replace('-', '_'), underline_dict(v)) for k, v in six.iteritems(d))
    
    
    if __name__ == "__main__":
        dict_test_1 = "FINALFANTASY"
        dict_test_2 = {"Nintendo-GF": "Pokemon"}
        dict_test_3 = {"Nintendo-part": {"GF-part": "Pokemon"}}
    
        print("1", underline_dict(dict_test_1))
        print("2", underline_dict(dict_test_2))
        print("3", underline_dict(dict_test_3))
    

    结果为
    

    可见该函数的作用是，若字典的key值包含-则将其转换为_，其余不变

  • 将处理完后产生的config赋值给ctx.default_map，用于设置指定command的指定option的默认值，形式为{command: {option:value}}，返回config

• connect_db

  • 具体实现

    def connect_db(ctx, param, value):
        if not value:
            return
        return utils.Get(lambda: connect_database(value))
    

  • pyspider.libs.utils.Get的具体实现

    
    class Get(object):
        """
        Lazy value calculate for object
        """
    
        def __init__(self, getter):
            self.getter = getter
    
        def __get__(self, instance, owner):
            return self.getter()
    

    Get作为data descriptor，必须产生实例且必须成为为另一个类的类属性，所以connect_db返回的结果将成为某个类的类属性
  
  

  • pyspider.database.connect_database的具体实现

    def connect_database(url):
    """
    create database object by url
    
    mysql:
        mysql+type://user:passwd@host:port/database
    sqlite:
        # relative path
        sqlite+type:///path/to/database.db
        # absolute path
        sqlite+type:path/to/database.db
        # memory database
        sqlite+type://
    mongodb:
        mongodb+type://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]]
        more: http://docs.mongodb.org/manual/reference/connection-string/
    sqlalchemy:
        sqlalchemy+postgresql+type://user:passwd@host:port/database
        sqlalchemy+mysql+mysqlconnector+type://user:passwd@host:port/database
        more: http://docs.sqlalchemy.org/en/rel_0_9/core/engines.html
    redis:
        redis+taskdb://host:port/db
    elasticsearch:
        elasticsearch+type://host:port/?index=pyspider
    local:
        local+projectdb://filepath,filepath
    
    type:
        taskdb
        projectdb
        resultdb
    
    """
    db = _connect_database(url)
    db.copy = lambda: _connect_database(url)
    return db
    

    该函数用来创建数据库对象，url的具体要求如注释所示
    
    
    
    

    其中pyspider.database._connect_database的具体实现为

    def _connect_database(url):  # NOQA
        parsed = urlparse(url)
    
        scheme = parsed.scheme.split('+')
        if len(scheme) == 1:
            raise Exception('wrong scheme format: %s' % parsed.scheme)
        else:
            engine, dbtype = scheme[0], scheme[-1]
            other_scheme = "+".join(scheme[1:-1])
    
        if dbtype not in ('taskdb', 'projectdb', 'resultdb'):
            raise LookupError('unknown database type: %s, '
                            'type should be one of ["taskdb", "projectdb", "resultdb"]', dbtype)
    
        if engine == 'mysql':
            return _connect_mysql(parsed,dbtype)
    
        elif engine == 'sqlite':
            return _connect_sqlite(parsed,dbtype)
        elif engine == 'mongodb':
            return _connect_mongodb(parsed,dbtype,url)
    
        elif engine == 'sqlalchemy':
            return _connect_sqlalchemy(parsed, dbtype, url, other_scheme)
    
    
        elif engine == 'redis':
            if dbtype == 'taskdb':
                from .redis.taskdb import TaskDB
                return TaskDB(parsed.hostname, parsed.port,
                            int(parsed.path.strip('/') or 0))
            else:
                raise LookupError('not supported dbtype: %s', dbtype)
        elif engine == 'local':
            scripts = url.split('//', 1)[1].split(',')
            if dbtype == 'projectdb':
                from .local.projectdb import ProjectDB
                return ProjectDB(scripts)
            else:
                raise LookupError('not supported dbtype: %s', dbtype)
        elif engine == 'elasticsearch' or engine == 'es':
            return _connect_elasticsearch(parsed, dbtype)
    
        else:
            raise Exception('unknown engine: %s' % engine)
    

    变量parsed中含有属性scheme, netloc, path, params, query, username, password，host, port等
    其中scheme为://前面的部分，netloc为://后面的部分到/为止，path为从/开始的部分

    
    

    engine表示采用的数据库，dbtype为数据库的用途，只能在["taskdb", "projectdb", "resultdb"]中选择，other_scheme表示另一种数据库方案(如redis+mongodb+taskdb, mongodb+taskdb为other_scheme)，其中redis仅支持taskdb，local仅支持project。

    
    

    所有连接数据库的操作流程都是先解析parsed，之后根据传入的dbtype进行数据库初始化等相关操作，暂时不展开叙述

• load_cls

  • 具体实现

    def load_cls(ctx, param, value):
        if isinstance(value, six.string_types):
            return utils.load_object(value) 
        return value
    

  • if语句用于python2，python3的兼容，之后出现与six模块相关的语句一般都是这个作用，除特殊情况之后不在说明。

  • load_object具体实现

    if "." not in name:
        raise Exception('load object need module.object')
    
    module_name, object_name = name.rsplit('.', 1)
    if six.PY2:
        module = __import__(module_name, globals(), locals(), [utf8(object_name)], -1)
    else:
        module = __import__(module_name, globals(), locals(), [object_name])
    return getattr(module, object_name)
    

  • 最终返回module.object

• connect_rpc
  • 具体实现

    def connect_rpc(ctx, param, value):
        if not value:
            return
        try:
            from six.moves import xmlrpc_client
        except ImportError:
            import xmlrpclib as xmlrpc_client
        return xmlrpc_client.ServerProxy(value, allow_none=True)
    

  • 连接远程服务器，使用XML-RPC协议