scrapy+redis mongo存储

​ 分布式 item 数据存在 redis 中。使用以上代码将 redis 中的 item 结果数据存入 mongo 数据库中。代码测试可用，具体细节部分需要与个人开发环境有关，请自行调整。

import redis,pymongo,json

# redis 配置项(请自行替换)
redis_url = '10.15.112.21'
redis_port = 6379
redis_db = 2
redis_passwd = 'a11112222'

# mongo 配置项(请自行替换)
mongo_url = '192.168.12.140'
mongo_port = 27017
mongo_dbname = 'dbname'
mongo_table = 'tablename'
mongo_user = 'username'
mongo_passwd = 'password'



def process_item():
    redis_cli = redis.StrictRedis(host=redis_url, port=redis_port, db=redis_db, password=redis_passwd)
    mongo_cli = pymongo.MongoClient(host=mongo_url, port=mongo_port, username=mongo_user, password=mongo_passwd)
    mongo_DB = mongo_cli[mongo_dbname]
    mongo_TB = mongo_DB[mongo_table]
    while True:
        try:
            source, data_json = redis_cli.blpop('wanbiao:items')
            # 
            data = json.loads(data_json.decode('utf-8'))
        except:
            print('redis 获取数据失败')
            break
        try:
            mongo_TB.insert(data)
        except:
            print('mongo插入失败')
            break
        
        
if __name__ == '__main__':
    process_item()