python爬虫，爬取一系列新闻

这个作业的要求来自于：https://edu.cnblogs.com/campus/gzcc/GZCC-16SE2/homework/2941。

由于存在多次请求，所以稍微将请求封装如下

def tranfrom_dom_tree(url):
    '''
        将获取的html文本转化为dom树
    '''
    response = requests.get(url);
    response.encoding = "utf-8";
    return BeautifulSoup(response.text, "html.parser");

 

将具体新闻内容封装如下

class News(object):
    '''
        广商校园新闻数据模型
    '''
    def __init__(self, url):
        self._url = url;             #新闻网页地址
        self._dom_tree = tranfrom_dom_tree(url);
        self._show_infos = self._dom_tree.select(".show-info")[0].text.split();
        self._update_time = "";
        self._auditor = "";
        self._auothor = "";
        self._origin = "";

        for index, args in enumerate(self._show_infos):
            if args.startswith("发布时间"):
                self._update_time = args[5:] + " " + self._show_infos[index+1];
                continue;
            elif args.startswith("作者"):
                self._auothor = args[3:];
                continue;
            elif args.startswith("审核"):
                self._auditor = args[3:];
                continue;
            elif args.startswith("来源"):
                self._origin = args[3:];
                continue;

    @property
    def title(self):
        '''
            :return:  新闻标题
        '''
        return self._dom_tree.select(".show-title")[0].text;

    @property
    def auothor(self):
        '''
            :return:  新闻作者
        '''
        return self._auothor;

    @property
    def auditor(self):
        '''
            :return:  新闻审核
        '''
        return self._auditor;

    @property
    def origin(self):
        '''
            :return:  新闻发布单位
        '''
        return self._origin;

    @property
    def update_time(self):
        '''
            :return:  新闻最后更新时间
        '''
        return self._update_time;

    @update_time.setter
    def update_time(self, time):
        '''
            设置最后更新时间
            :param time:  时间
        '''
        self._update_time = time;

    @property
    def times(self):
        '''
            :return:  点击次数
        '''
        clickUrl = 'http://oa.gzcc.cn/api.php?op=count&id={}&modelid=80'.format(self.news_id);
        response = requests.get(clickUrl);
        click = re.findall('(\d+)', response.text)[-1];
        return click;

    @property
    def news_id(self):
        '''
            :return:  新闻标识
        '''
        time = datetime.strptime(self._update_time, '%Y-%m-%d %H:%M:%S');
        time = time.strftime("%m%d");
        return re.match('http://news.gzcc.cn/html/.*/.*/(\d+).html', self._url).group(1);

    @property
    def summary(self):
        '''
            :return:  新闻摘要内容
        '''
        return self._summary;

    @summary.setter
    def summary(self, text):
        '''
            设置新闻摘要
            :param text:  新闻摘要
        '''
        self._summary = text;

    def to_dict(self):
        '''
            将此类实例转换为字典
            :return: 转换后的字典
        '''
        dict = {};
        dict["news_url"] = self._url;
        dict["news_id"] = self.news_id
        dict["news_title"] = self.title;
        dict["news_summary"] = self.summary;
        dict["news_update_time"] = self.update_time;
        dict["news_times"] = self.times;
        dict["news_auothor"] = self.auothor;
        dict["news_auditor"] = self.auditor;
        dict["news_origin"] = self.origin;
        return dict;

 

　　

对新闻进行批操作代码如下

class GZCCNewsReptile(object):
    '''
        广州商学院校园新闻获取工具
    '''
    def __init__(self):
        self._news_type = "dict";
        self._root_url = "http://news.gzcc.cn/html/xiaoyuanxinwen/";
        self._url = self.page_url();
        self._dom_tree = tranfrom_dom_tree(self._url);

    def page_url(self, page=1):
        '''
            将指定页面解析数值解析对应url
            :param page:  指定的一个新页面
        '''
        if page == 1:
            self._now_page = "index";
        else:
            self._now_page = page;
        return self._root_url+str(self._now_page)+".html";

    @property
    def count(self):
        '''
            :return:  返回校园新闻总条数
        '''
        count = self._dom_tree.select(".a1")[0].text;
        return int(count[0:-1]);

    @property
    def page(self):
        '''
            :return:  获取校园新闻总页数
        '''
        page = int(self.count) / 10;
        int_page = int(page);
        if page > int_page:
            return ( int_page + 1);
        else:
            return int_page;

    def get_news_from_news_page_size(self, start_page, end_page):
        '''
            设置爬取页数范围,
            start_page 小于 1 抛出    异常
            start_page 大于 end_page 抛出 异常
            end_page 大于 总页数 抛出 异常
            :param start_page:  要爬取范围的开始页
            :param end_page:    结束爬取范围的页面(不包括该页面)
        '''
        if start_page < 1:
            raise IndexError("start_page不在指定范围内");
        if start_page > end_page:
            raise IndexError("start_page大于end_page");
        if end_page > self.page:
            raise IndexError("end_page不在指定范围内");
        news_list = [];
        times = (index for index in range(start_page, end_page));
        #  爬取指定范围数据
        for index in times:
            #  news_page_list = self.get_news_from_page_url(self.page_url(index));
            news_page_list = self.get_page_news(index);  #  较上句更利于封装
            news_list.append(news_page_list);
        news_list = sum(news_list, []);
        return news_list;

    def get_page_news(self, page):
        '''
            获取指定页数
            若指定的页数在可爬取页数的范围之外，则抛出运行异常异常
            :param pages:  指定的页数
            :return: 返回指定页的新闻列表
        '''
        if page < 1 or page > self.count:
            raise IndexError("page不在指定范围内");
        else:
            print("\r当前正在%d页" % page, end="");
            return self.get_news_from_page_url(self.page_url(page));

    def get_news_from_page_url(self, url):
        '''
            获取指定url的所有新闻列表
            :param pages:  指定的页面url
            :return:  news_page_list的列表
        '''
        dom_tree = tranfrom_dom_tree(url);
        news_ui = dom_tree.select(".news-list li a");
        news_page_list = [];
        for index in range(0, 10):
            try:
                a_tag = news_ui[index];
                href = a_tag.get("href");

                news = News(href);
                news.summary = a_tag.select(".news-list-description")[0].text;
                news_page_list.append(eval("news.to_"+self.news_type+"()"));
            except Exception:
                error_log = "此页面不正常: %s"%href;
                print("此页面不正常: %s"%href, end="");
                with open("./logger.txt", "a", encoding="utf-8") as file:
                    now_time = time.time();
                    file.write(error_log + " 错误时间:" + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(now_time)) );

        return news_page_list;

    @property
    def news_type(self):
        '''
            新闻默认类型设置
            :return: 返回默认类型
        '''
        return self._news_type;
    @news_type.setter
    def news_type(self, type="dict"):
        '''
            新闻可选类型
            :param type:  选择的类型
        '''
        if type == "dict":
            self._news_type = type;
        else :
            raise Exception("未能匹配该类型");

测试代码如下

    #  测试GZCCNewsReptil可用性
    start_page = 106
    # news_list = GZCCNewsReptile().get_news_from_news_page_size(start_page,start_page+10);
    news_list = GZCCNewsReptile().get_news_from_news_page_size(1,  256);
    pandas_date = pandas.DataFrame(news_list);
    print(news_list);
    # #csv
    pandas_date.to_csv(".\pandas_date.csv", encoding="utf-8_sig");
    # #sql
    # with sqlite3.connect("test.sqlite") as db:
    #     pandas_date.to_sql("test", db);
    #     date = pandas_date.read_sql_query('SELECT * FROM gzccnewsdb5', con=db)
    #     date[date["news_times"]>380];

---

 

转载于:https://www.cnblogs.com/destinymingyun/p/10674744.html