scrapy_redis分布式爬虫 从redis写到mysql数据库中

import redis
import MySQLdb
import json

def process_item():
# 创建redis数据库连接
rediscli = redis.Redis(host = “127.0.0.1”, port = 6379, db = 0)

# 创建mysql数据库连接
mysqlcli = MySQLdb.connect(host = "127.0.0.1", port = 3306, \
    user = "", passwd = "", db = "数据库")

offset = 0

while True:
    # 将数据从redis里pop出来
    source, data = rediscli.blpop("表名:items")
    item = json.loads(data)
    try:
        # 创建mysql 操作游标对象，可以执行mysql语句
        cursor = mysqlcli.cursor()

        cursor.execute("insert into 表名 (username, age) values (%s, %s)", [item['username'], item['age']])
        # 提交事务
        mysqlcli.commit()
        # 关闭游标
        cursor.close()
        offset += 1
        print offset
    except:
        pass

if name == “main“:
process_item()