python爬虫豆瓣top250 并写入数据库

获取豆瓣top250页面html

• 因为豆瓣有反爬机制，所以试了各种方法，最终采用模拟浏览器发送请求
• 在requests做请求的时候，为了避免ssl认证，可以将verify=False

import requests
from bs4 import BeautifulSoup
import re
from faker import Faker
from database.dbc import Pymysql_dbc

def getHTMLText(url):
    faker = Faker()
    headers = {
   
        'User-Agent': faker.user_agent(),
        'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"
    }
    # 禁用报错信息
    requests.packages.urllib3.disable_warnings()
    response = requests.get(url, headers=headers, verify=False)
    if response.status_code == 200:
        html = BeautifulSoup(response