python多线程爬取_python多线程爬取网页

#-*- encoding:utf8 -*-

'''

Created on 2018年12月25日

@author: Administrator

'''

from multiprocessing.dummy import Pool as pl

import csv

import requests

from lxml import etree

def spider(url):

header = {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 \

(KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36"}

r = requests.get(url=url, headers=header)

return r.json()

def spider_detail(url):

resp = spider(url)

title = resp.get('data').get('title')

print(title)

content = resp.get('data').get('content')

try:

title_clear = title.replace('|', '').replace('?', '')

content_clear = content.replace('

','\n\n').replace('

','')

sel = etree.HTML(content_clear)

content_clear = sel.xpath('string(//*)')

artical_write(title_clear, content_clear)

print(title_clear)

except:

pass

def get_all_urls(page_number):

for i in range(1, page_number + 1):

url = 'https://36kr.com/api/search-column/mainsite?per_page=20&page=' + str(i)

resp = spider(url)

artical_data = resp.get('data').get('items')

for url_data in artical_data:

number = url_data.get('id')

artical_url = 'https://36kr.com/api/post/'+ str(number) +'/next'

yield artical_url

def artical_write(title, content):

with open('d:/spider_data/11.11/' + title + '.txt', 'wt', encoding='utf-8') as f:

f.write(content)

if __name__ == '__main__':

# 线程数， 默认为cpu核心数

pool = pl(4)

# url列表收集

all_url = []

for url in get_all_urls(100):

all_url.append(url)

# 多线程爬取

pool.map(spider_detail, all_url)

pool.close()

pool.join()