爬虫小练习：猫眼电影TOP100资料爬取（正则--基础模块到多进程、多线程）

# from multiprocessing import Pool
from threading import Thread
import requests
import re
import json

def get_page(url):
	headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'}
	try:
		response = requests.get(url,headers=headers)
		if response.status_code == 200:
			return response.text
		return None
	except:
		return None

def parse_page(html):
	pattern = re.compile('<dd>.*?board-index.*?>(\d+)</i>.*?data-src="(.*?)".*?name">.*?">(.*?)</a>.*?star">(.*?)</p>.*?releasetime">(.*?)</p>.*?<i class="integer">(.*?)</i>.*?fraction">(.*?)</i>.*?</dd>',re.S)
	content = re.findall(pattern,html)
	# print(content)
	for item in content:
		yield {
			'index':item[0],
			'image':item[1],
			'name':item[2],
			'star':item[3].strip()[3:],
			'time':item[4].strip()[5:],
			'score':item[5]+item[6]
		}

def write_data(items):
	with open('movie_detail.txt','a',encoding='utf-8') as f:
		f.write(json.dumps(items,ensure_ascii=False) + '\n')
		f.close()

def main(offset):
	url = 'http://maoyan.com/board/4?offset=%s'%(offset)
	html = get_page(url)
	# print(html)
	contents = parse_page(html)
	for items in contents:
		# print(items)
		# print(type(items))
		write_data(items)

if __name__ == '__main__':
	# for offset in range(10):
	# 	main(str(offset*10))

	# pool = Pool()
	# pool.map(main,[offset*10 for offset in range(10)])

	for offset in range(10):
		t = Thread(target=main,args=(offset*10,))
		# t.setDaemon(True)
		t.start()