Python爬取学习猿地文章标题，链接，时间，作者

 

 

'''
爬取网址：学习猿地猿圈
爬取内容：文章标题，文章连接，作者，时间
工具：bs4,requests
结果：爬取到文件之中
'''
from bs4 import BeautifulSoup
import requests,json
#1.定义请求头和url
url = 'https://www.lmonkey.com/t'
headers = {
     "User-Agent":  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3100.0 Safari/537.36"
}
alldata = []
#2.请求获取数据
res = requests.get(url = url,headers = headers)
if res.status_code == 200:
 
#3.解析数据
    soup = BeautifulSoup(res.text,'lxml')
#获取页面中所有的文章
    divclass = soup.find_all( 'div',class_="list-group-item list-group-item-action p-06")
    for i in divclass:
        my_title = i.find('div',class_="topic_title mb-0 lh-180")
        if my_title:
            my_title = my_title.text.split("\n")[0]
            my_url = i.a["href"]
            my_time = i.span['title']
            my_author = i.strong.a.text
            print(my_author)
            print(my_url)
            print(my_time)
            print(my_title)
            temp = {'title':my_title,
                    'passage_url':my_url,
                    'time':my_time,
                    'author':my_author
                    }
            alldata.append(temp)
            #4.写入文件
    with open('C:/Users/lsy/Desktop/lsy.json','w') as fp:
        json.dump(alldata,fp)