redis数据库到mysql或mongodb数据库

# -*- coding:utf-8 -*-
# item_mongodb.py

import redis
import pymongo
import json


def main():
    redis_connect = redis.StrictRedis(host='127.0.0.1', port=6379, db=0)
    mongo_connect = pymongo.MongoClient(host='127.0.0.1', port=27017)

    db = mongo_connect['sina']
    collection = db['sina_articles']

    while True:
        source, data = redis_connect.blpop(['sina:items'])
        item = json.loads(data)
        collection.insert(item)


if __name__ == '__main__':
    main()

# -*- coding:utf-8 -*-
# item_mysql.py
import MySQLdb
import mysql.connector
import redis
import json


def main():
    redis_connect = redis.StrictRedis(host='localhost', password='123456', port=6379, db=0)
    mysql_connect = mysql.connector.connect(host='127.0.0.1', port=3306, user='root', passwd='123456', database='sina', use_unicode=True)


    while redis_connect.exists('sina:items'):
        source, data = redis_connect.blpop('sina:items')
        item = json.loads(data, encoding='utf-8')
        print (item)
        cur = mysql_connect.cursor()
        cur.execute(
            'insert into sina_articles(parent_title, parent_url, sub_title, sub_url, sub_path, file_url, '
            'file_title, file_content, crawled, spider) values(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)',
            (item['parent_title'], item['parent_url'], item['sub_title'], item['sub_url'], item['sub_path'],
             item['file_url'], item['file_title'], item['file_content'], item['crawled'], item['spider']))
        mysql_connect.commit()
        cur.close()
    mysql_connect.close()


if __name__ == "__main__":
    main()

# -*- coding:utf-8 -*-
# redis数据库 到 Excel
import redis
import json
from openpyxl import Workbook


def main():
    # redis_connect = redis.StrictRedis(host='localhost', port=6379, db=0)
    redis_connect = redis.StrictRedis(host='localhost', password='123456', port=6379, db=0)
    a = 1
    while redis_connect.exists('hungry:items'):
        source, data = redis_connect.blpop('hungry:items')
        item = json.loads(data, encoding='utf-8')
        print (item)
        a =a + 1
        ws.append([item['id'],
                   item['classification'],
                   item['shopName'],
                   item['address'],
                   item['foodInformation'][0]['rating'],
                   item['foodInformation'][0]['price'],
                   int(item['foodInformation'][0]['sale']),
                   int(item['foodInformation'][0]['Evaluation']),
                   item['foodInformation'][0]['foodName'],
                   ])
    wb.save('nba.xlsx')  # 保存文件
    print ('a:', a)


if __name__ == "__main__":
    wb = Workbook()  # class实例化
    ws = wb.active  # 激活工作表
    ws.title = "New Shit"
    title_list = ['id', 'classification', 'shopName', 'address', 'rating', 'price', 'Evaluation', 'foodName', 'sale']
    ws['A1'] = '饿了么'
    ws.append(title_list)  # 添加一行数据

    main()

 

转载于:https://www.cnblogs.com/Mint-diary/p/9752841.html