本人的第一次爬虫,爬取后的文章保存在txt文件里。
参考:python3 网络爬虫开发实战
import requests
from requests import RequestException
from pyquery import PyQuery as pq
url = 'https://www.zhihu.com/explore'
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit'
'/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'
}
def get_one_page(url):
try:
response = requests.get(url, headers=headers)
if response.status_code == 200:
return response.text
return None
except RequestException:
return None
def write_result(content):
doc = pq(content)
items = doc('.explore-tab .feed-item').items()
with open('explore.txt','a',encoding='utf-8') as file:
for item in items:
question = item.find('h2').text()