爬虫感悟2

import requests
from requests.exceptions import RequestException
import re
import json
def getHTMLText(url):
    try:
        response = requests.get(url, headers={'user-agent': 'Mozilla/5.0'})
        if response.status_code == 200:
            return response.text
    except RequestException:
        return None

def parseHTMLText(html):
    pattern = re.compile('
   
   

    
    .*?board-index.*?>(\d+).*?data-src="(.*?)".*?name">
    
    
   
   
', re.S)
    items = re.findall(pattern,html)
    for item in items:
        yield{
            "index": item[0],
            "image": item[1],
            "title": item[2],
            "stars": item[3].strip()[3:],
            "time": item[4].strip()[5:],
            "score": item[5]+item[6]
        }

def writeHTMLText(content):
    with open('result.text','a',encoding='utf-8') as f:
        f.write(json.dumps(content,ensure_ascii=False) + '\n')
        f.close()

def main(page):
    url = 'http://maoyan.com/board/4?' + 'offset=' + str(page*10)
    html = getHTMLText(url)
    for item in parseHTMLText(html):
        print(item)
        writeHTMLText(item)

    '''item = parseHTMLText(html)
    print(item)'''

if __name__ == '__main__':
    for i in range(10):
        main(i)

一开始equests的访问被拒绝，然后重新改变了headers

引入requests库的requests.exceptions,使问题更具体化

第二个函数使用了生成器，减少内存的使用

读写文件时加上了 encoding='utf-8’使生成的文件显示中文