Python爬虫学习笔记（十一）

# import sys
# n=int(sys.argv[1])
# with open('dicts.txt') as f:
#     ele = f.readlines()







####################################
#爬取猫眼电影排行数据
import requests
import re
import time
import json
from requests.exceptions import RequestException
def get_page(url):
    try:
        headers = {
            "user-agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3947.100 Safari/537.36"}
        resp = requests.get(url=url,headers=headers)
        if resp.status_code == 200:
            return resp.text
        return None     ###和break的效果一样 退出整个函数   而pass指的是退出当前循环
    except RequestException:
        return None
def parse_page(resp):
    ####  看network里面response的html内容  element里面可能经过js渲染不是真实的
    ####   正则表达式 .*？ 的重要性 如果直接打两个标签 如果两个标签之间有空格没打出来 也匹配不到  尽量多用.*？
    ####   因为自己写可能漏掉空格
    pattern  = re.compile('<dd>.*?board-index.*?>(\d+)</i>.*?data-src="(.*?)"'
                         + '.*?alt="(.*?)".*?<p class="star">(.*?)</p>'
                         +'.*?releasetime">(.*?)</p>.*?'
                         +'integer">(.*?)</i>.*?'
                         +'fraction">(.*?)</i>',re.S)  ###re.S的作用是让，匹配包括换行符在内的所有字符 因为html文本很多是有换行的
    # pattern = re.compile('<dd>.*?board-index.*?>(\d+)</i>.*?data-src="(.*?)".*?name"><a'
    #                          + '.*?>(.*?)</a>.*?star">(.*?)</p>.*?releasetime">(.*?)</p>'
    #                          + '.*?integer">(.*?)</i>.*?fraction">(.*?)</i>.*?</dd>', re.S)
#     #   <dd>
#                         <i class="board-index board-index-1">1</i>
#     <a href="/films/13824" title="射雕英雄传之东成西就" class="image-link" data-act="boarditem-click" data-val="{movieId:13824}">
#       <img src="//s3plus.meituan.net/v1/mss_e2821d7f0cfe4ac1bf9202ecf9590e67/cdn-prod/file:5788b470/image/loading_2.e3d934bf.png" alt="" class="poster-default" />
#       <img data-src="https://p0.meituan.net/movie/53d18b640f8af0c47d7c95488450c265388754.jpg@160w_220h_1e_1c" alt="射雕英雄传之东成西就" class="board-img" />
#     </a>
#     <div class="board-item-main">
#       <div class="board-item-content">
#               <div class="movie-item-info">
#         <p class="name"><a href="/films/13824" title="射雕英雄传之东成西就" data-act="boarditem-click" data-val="{movieId:13824}">射雕英雄传之东成西就</a></p>
#         <p class="star">
#                 主演：张国荣,梁朝伟,张学友
#         </p>
# <p class="releasetime">上映时间：1993-02-05(中国香港)</p>    </div>
#     <div class="movie-item-number score-num">
# <p class="score"><i class="integer">8.</i><i class="fraction">8</i></p>
#     </div>
#
#       </div>
#     </div>
#
#                 </dd>
    #
    #
    items = re.findall(pattern,resp)   ###一个包含很多元组的列表
    ####其中一个元组
    #('1', 'https://p0.meituan.net/movie/53d18b640f8af0c47d7c95488450c265388754.jpg@160w_220h_1e_1c',
    #  '射雕英雄传之东成西就', '\n           主演：张国荣,梁朝伟,张学友\n        ',
    # '上映时间：1993-02-05(中国香港)', '8.', '8')

    ####  一个元组生成一个字典
    ####  一个页面循环返回多个字典
    for item in items:
        yield{
            'index':item[0],
            'image':item[1],
            'title':item[2],
            'actor':item[3].strip()[3:] if len(item[3])>3 else '',  #strip()函数去掉多余的空格或换行符
            #[3:] 从下标3开始 【0：】代表从下表0开始 只要第三个字符 以后 的字符 下标0123 即从第四个字符开始取  去掉 主演：这三个字符 只要演员名字
            #if len(item[3])>3 else'' 只有item[3]大于三个字符才执行前面的语句 否则‘’保证安全不空
            'time':item[4].strip()[5:] if len(item[4])>5 else '',
            'score':item[5].strip()+item[6].strip()


        }
    #yield 相当于return 在程序中返回某个值
def write_to_file(item):
    with open('catmovie.txt','a',encoding='utf-8') as fp:
        fp.write(json.dumps(item,ensure_ascii=False)+'\n') #json.dumps()函数是将字典转化为字符串
        #json.loads()函数是将字符串转化为字典
def main():
    #https://maoyan.com/board/4?offset=10
    for i in range(2):
        offset = 10*i
        url = 'https://maoyan.com/board/4?offset='+str(offset)
        resp = get_page(url)#获取网页的返回结果
        items = parse_page(resp)#解析网页的返回结果 yield{}返回的
        #print('parse_page返回的解析结果 ：'+str(items))   #<generator object parse_page at 0x03CDA4F0>
        time.sleep(1)
        for item in items:
            print(item)
            write_to_file(item)#持久化存储
if __name__=="__main__":

    main()
###################################################################################################
#github
# import json
# import requests
# from requests.exceptions import RequestException
# import re
# import time
#
#
# def get_one_page(url):
#     try:
#         headers = {
#             "user-agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3947.100 Safari/537.36"}
#         response = requests.get(url, headers=headers)
#         if response.status_code == 200:
#             return response.text
#         return None
#     except RequestException:
#         return None
#
#
# def parse_one_page(html):
#     pattern = re.compile('<dd>.*?board-index.*?>(\d+)</i>.*?data-src="(.*?)".*?name"><a'
#                          + '.*?>(.*?)</a>.*?star">(.*?)</p>.*?releasetime">(.*?)</p>'
#                          + '.*?integer">(.*?)</i>.*?fraction">(.*?)</i>.*?</dd>', re.S)
#     items = re.findall(pattern, html)
#     for item in items:
#         yield {
#             'index': item[0],
#             'image': item[1],
#             'title': item[2],
#             'actor': item[3].strip()[3:],
#             'time': item[4].strip()[5:],
#             'score': item[5] + item[6]
#         }
#
#
# def write_to_file(content):
#     with open('maoyanmoviesresult.txt', 'a', encoding='utf-8') as f:
#         f.write(json.dumps(content, ensure_ascii=False) + '\n')
#
#
# def main(offset):
#     url = 'http://maoyan.com/board?offset=' + str(offset)
#     html = get_one_page(url)
#     for item in parse_one_page(html):
#         print(item)
#         write_to_file(item)
#
#
# if __name__ == '__main__':
#     for i in range(10):
#         main(offset=i * 10)
#         time.sleep(1)