import requests
from requests.exceptions import RequestException
import re
import json
def getHTMLText(url):
try:
response = requests.get(url, headers={'user-agent': 'Mozilla/5.0'})
if response.status_code == 200:
return response.text
except RequestException:
return None
def parseHTMLText(html):
pattern = re.compile('
.*?board-index.*?>(\d+).*?data-src="(.*?)".*?name">
', re.S)
items = re.findall(pattern,html)
for item in items:
yield{
"index": item[0],
"image": item[1],
"title": item[2],
"stars": item[3].strip()[3:],
"time": item[4].strip()[5:],
"score": item[5]+item[6]
}
def writeHTMLText(content):
with open('result.text','a',encoding='utf-8') as f:
f.write(json.dumps(content,ensure_ascii=False) + '\n')
f.close()
def main(page):
url = 'http://maoyan.com/board/4?' + 'offset=' + str(page*10)
html = getHTMLText(url)
for item in parseHTMLText(html):
print(item)
writeHTMLText(item)
'''item = parseHTMLText(html)
print(item)'''
if __name__ == '__main__':
for i in range(10):
main(i)
一开始equests的访问被拒绝,然后重新改变了headers
引入requests库的requests.exceptions,使问题更具体化
第二个函数使用了生成器,减少内存的使用
读写文件时加上了 encoding='utf-8’使生成的文件显示中文