Homework Day07

最新推荐文章于 2019-08-26 08:43:00 发布

晚颜

最新推荐文章于 2019-08-26 08:43:00 发布

阅读量403

点赞数

分类专栏： Python作业

本文链接：https://blog.csdn.net/ZZDPP/article/details/100025503

版权

Python作业专栏收录该内容

7 篇文章 0 订阅

订阅专栏

爬取清华大学中新闻中心的所有网页代码

import requests
import threading
import re

class QHDX(threading.Thread):
    def __init__(self, start_, end_,lock):
        threading.Thread.__init__(self)
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36'
        }
        self.base_url = 'http://news.tsinghua.edu.cn/publish/thunews/index.html'
        self.start_ = start_
        self.end_ = end_
        self.lock = lock
        print(self.base_url)
    def run(self):
        for offset in range(self.start_, self.end_, 50):
            url = self.base_url
            response = requests.get(url, headers=self.headers)
            response.encoding = 'utf8'
            html = response.text
            info_list = self.get_Information(html)
            with self.lock:
                self.write(info_list)
            print('offset {} OK !'.format(offset))

    def get_Information(self, html):
        information_list = []
        for line in html.split('\n'):
            if '<li><a href="' in line:
                information_list.append(line)
        return information_list

    def write(self,info_list):
        str_ = str(info_list) + '\n'
        with open('res.txt',mode='a',encoding='utf8') as file:
            file.write(str_)

if __name__ == "__main__":
    threads = []
    lock = threading.Lock()
    for i in range(1):
        t = QHDX(start_=i * 50, end_=(i + 1) * 50,lock=lock)
        t.start()
        threads.append(t)
        for j in threads:
            t.join()

    print('Over')