（21）爬虫beautiful soup库 实战

import requests
from fake_useragent import UserAgent
from time import sleep
from random import randint
from bs4 import BeautifulSoup

def get_html(url):
    headers={
        'User-Agent':UserAgent().firefox
    }
    proxies = {
        "http": "http://35.236.158.232:8080"
    }
    sleep(randint(3,9))
    html_response=requests.get(url,headers=headers,proxies=proxies)
    html_response.encoding = 'utf-8'

    if html_response.status_code == 200:
        return html_response.text
    else:
        print(html_response.status_code)

def parse_html(html):
    soup = BeautifulSoup(html,'lxml')
    name = soup.select('div.book-info > h1 > em')[0].text
    all_info={
        'names': name
    }
    return all_info

def parse_get_url(index_url):
    html_r = get_html(url = index_url)
    soup = BeautifulSoup(html_r,'lxml')
    all_a = soup.select('div.name-box > a.name')
    all_url=[]
    for a in all_a:
        all_url.append(a.attrs['href'])
    return ['https:{}'.format(movie_url) for movie_url in all_url]

def main():
    index_url = 'https://www.qidian.com/rank'
    print(index_url)
    movie_url_list = parse_get_url(index_url=index_url)
    print(movie_url_list)
    for movie_url in movie_url_list:
        response = get_html(movie_url)
        outcome = parse_html(response)
        print(outcome)

if __name__ == '__main__':
    main()
    print('爬虫程序结束')