使用python3的BeautifulSoup爬取搜狐娱乐八卦新闻。
包括八卦标题、链接、更新时间,还有更多属性就不一一列举了
声明:这是简单demo 大神请绕道
#!/usr/bin/python3
from bs4 import BeautifulSoup
import requests
import time as tie
print("爬取搜狐娱乐->八卦新闻 ;栏目标题、链接、时间")
def getall(urls):
for url in urls:
print(url)
data = requests.get(url)
data.encoding = "gb18030"
soup = BeautifulSoup(data.text, "lxml")
titles = soup.select("body > div.scola_top_bg > div.main.area > div.lc > div.f14list > ul > li > a")
times = soup.select(
"body > div.scola_top_bg > div.main.area > div.lc > div.f14list > ul > li > span:nth-of-type(1)")
# print(names)
info = []
for title, time in zip(titles, times):
dataMap = {
"title": title.get_text(),
"href": title.get("href"),
"time": time.get_text()
}
print(dataMap)
tie.sleep(1)
url = ["http://yule.sohu.com/stars_news.shtml"]
urls = ["http://yule.sohu.com/stars_news_{}.shtml".format(str(i)) for i in range(291,193,-1)]
getall(url)
getall(urls)
更多分享或者需要其它资料可关注我个人微信公众“yyjuan”,在后台联系我