scrapy-redis从redis到kafka生产消费

redis到kafka生产者

# -*- coding: utf-8 -*-
import time
import json
import random
import htmlmin
from lxml import etree
from datetime import datetime
from redis import StrictRedis
from kafka import KafkaProducer
from multiprocessing import Process

# redis
redis_host = '172.16.30.45'
client = StrictRedis(
    host=redis_host, port=6379, db=0
)

#kafka Producer
kafka_host_1 = '172.16.30.25'
kafka_host_2 = '172.16.30.45'
kafka_host_3 = '172.16.30.65'
producer = KafkaProducer(
    value_serializer=lambda v: json.dumps(v).encode('utf-8'),
    bootstrap_servers=['{}:9092'.format(kafka_host_1),'{}:9092'.format(kafka_host_2),'{}:9092'.format(kafka_host_3)]
)

spider_names = ['onion_xs6qb_market_spider','onion_zj4o7_market_spider','onion_c2p3h_market_spider']


def dec2bin(n=0, max_len=8):
    """
    十进制转为对应的二进制字符串，首位不够补0
    :param n: 待转换的字节
    :param max_len: 需要转换为对应字节的长度
    :return:
    """
    tmp_bin = "{0:b}".format(n)
    if len(tmp_bin) < max_len:
        tmp_bin = "0"*(max_len-len(tmp_bin)) + tmp_bin
    return tmp_bin

def generate_data_id(producer_id=1, data_subtype=1, primary_id=4):
    """
    生成数据的data_id，2字节的producer_id+2字节的data_subtype+4字节的数据主键
    :param producer_id:
    :param data_subtype:
    :param primary_id:
    :return:
    """
    temp_str = dec2bin(producer_id, 16) + dec2bin(data_subtype, 16) + dec2bin(primary_id, 32)
    return int("0b"+temp_str, 2)


def transfer_redis_kafka(spider_name):
    while True:
        time.sleep(0.01)
        source, redis_data = client.blpop(["{}:items".format(spider_name)])
        redis_data = json.loads(redis_data.decode("utf-8"))
        response = etree.HTML(redis_data['raw_text'])
        contents = response.xpath("//html//body")[0].xpath("string(.)")
        kafka_data = {
            "data_id": generate_data_id(1,2, int(time.time())),
            "url": redis_data['url'],
            "content":"\n".join(contents),
            "crawl_time": redis_data['crawl_time'],
            "domain_name": redis_data['domain'],
            "title": redis_data['raw_title'],
            "html": htmlmin.minify(redis_data['raw_text'].encode('utf-8').decode('utf-8'), remove_all_empty_space=True),
            "language": redis_data['language'],
            "data_type":'webpage',
            "net_type": redis_data['net_type'],
            "h1": redis_data['h1'],
            "meta": redis_data['meta'],
            "headers": redis_data['headers'],
            "content_type": redis_data['content_type'],
            "content_encode": redis_data['content_encode'],
            "code": redis_data['code'],
            "links": redis_data['links'],
            "gmt_create": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S"),
            "gmt_modified": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S"),
        }
        print('web:',kafka_data)
        producer.send('demo.crawled_anwangzhongwen', kafka_data)
    # producer.close()

def task_schdule():
    processes = []
    for i, spider_name in enumerate(spider_names):
        process = Process(target=transfer_redis_kafka, args=(spider_name,))
        processes.append(process)
        processes[i].start()


if __name__ == '__main__':
    task_schdule()

kafka消费者

from kafka import KafkaConsumer

kafka_host_1 = '172.16.30.25'
kafka_host_2 = '172.16.30.45'
kafka_host_3 = '172.16.30.65'
consumer = KafkaConsumer('demo.crawled_anwangzhongwen',group_id="tor",
                         bootstrap_servers=['{}:9092'.format(kafka_host_1),'{}:9092'.format(kafka_host_2),'{}:9092'.format(kafka_host_3)],
                         auto_offset_reset='earliest',value_deserializer=json.loads
                         )

for data_kafka in consumer:
    print(data_kafka.value)