使用python爬取搜狐娱乐八卦新闻

使用python3的BeautifulSoup爬取搜狐娱乐八卦新闻。

包括八卦标题、链接、更新时间，还有更多属性就不一一列举了

声明：这是简单demo  大神请绕道

#!/usr/bin/python3
from bs4 import BeautifulSoup
import requests
import time as tie

print("爬取搜狐娱乐->八卦新闻 ；栏目标题、链接、时间")
def getall(urls):

    for url in urls:
        print(url)

        data = requests.get(url)
        data.encoding = "gb18030"
        soup = BeautifulSoup(data.text, "lxml")

        titles = soup.select("body > div.scola_top_bg > div.main.area > div.lc > div.f14list > ul > li > a")
        times = soup.select(
            "body > div.scola_top_bg > div.main.area > div.lc > div.f14list > ul > li > span:nth-of-type(1)")
        # print(names)
        info = []
        for title, time in zip(titles, times):
            dataMap = {
                "title": title.get_text(),
                "href": title.get("href"),
                "time": time.get_text()
            }
            print(dataMap)
        tie.sleep(1)


url = ["http://yule.sohu.com/stars_news.shtml"]
urls = ["http://yule.sohu.com/stars_news_{}.shtml".format(str(i)) for i in range(291,193,-1)]

getall(url)
getall(urls)

更多分享或者需要其它资料可关注我个人微信公众“yyjuan”，在后台联系我