今天自己做一个爬虫小练习,代码都正确但一直报错,很是无语
import requests
from bs4 import BeautifulSoup
if __name__ == "__main__":
url = 'https://www.shicimingju.com/book/hongloumeng.html'
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36'}
page_text = requests.get(url=url,headers=headers).text
soup = BeautifulSoup(page_text,'lxml')
li_list = soup.select('.book-mulu > ul>li')
fp = open('./honglou.txt','w',encoding='UTF-8')
for li in li_list:
title = li.a.string
detail_url ='http://www.shicimingju.com' + li.a['href']
#detail_page_text = requests.get(url=detail_url, headers=headers).text
detail_page_text = requests.get(url=detail_url, headers=headers).text
detail_soup = BeautifulSoup(detail_page_text,'lxml')
div_tag = detail_soup.find('div', class_='chapter_content')
#div_tag = detail_soup.find('div', class="chapter_content")
content = div_tag.text
#content = div_tag.text
fp.write(title + ':' + content + '\n')
print(title,'ok')
output:
Traceback (most recent call last):
File "E:/python/pythonProject/test/test.py", line 19, in <module>
content = div_tag.text
AttributeError: 'NoneType' object has no attribute 'text'
换了一个user-agent就好了,这是啥怪病??
Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36