Homewor07 _ 爬取清华新闻

在MySQL中创建表
CREATE DATABASE yn;
show databases;
use yn;
create table news(
-> id int(4) not null primary key auto_increment,
-> title char(20) not null);

import requests
import threading
import pymysql


class tsinghua(threading.Thread):
    def __init__(self, target_url, total_page, lock):
        threading.Thread.__init__(self)
        self.target_url = target_url
        self.total_page = total_page
        self.db = pymysql.connect(host='127.0.0.1',
                                  port=3306,
                                  user='root',
                                  password='forever1122',
                                  db='yn')
        self.id = 0
        self.lock = lock
    def run(self):
        #url_list = ['9648','10303','9649','9650']
        #for n in url_list:
        for page in range(1, self.total_page + 1):
            if page == 1:
                url = self.target_url.format('index')
            else:
                url = self.target_url.format('index_' + str(page))
            #with open('E:/python/Homework05/novel/title.txt',mode='a',encoding='utf8') as file:
                            #file.write(url+'\n')
            response = requests.get(url)
            response.encoding = 'utf8'
            html = response.text
            for line in html.split('\n'):
                if 'class="jiequ"' in line:
                    title = line.split('class="jiequ">')[1].split('</a>')[0]
                    #print(title)
                    with self.lock:
                        self.id += 1
                        sql = 'INSERT INTO news (id,title) values ({},"{}");'.format(
                            self.id, title)
                        self.write_to_mysql(sql)
                        print('[+] writ mysql over ~')
            print('[+] current page: {} ok'.format(page))

    def write_to_mysql(self, sql):
        cursor = self.db.cursor()
        try:
            cursor.execute(sql)
            self.db.commit()
        except Exception as e:
            print(e)
            self.db.rollback()

def Running():
    lock = threading.Lock()
    thread_1 = tsinghua('http://news.tsinghua.edu.cn/publish/thunews/9486/{}.html', 48, lock).start()
    thread_2 = tsinghua('http://news.tsinghua.edu.cn/publish/thunews/10303/{}.html', 548, lock).start()
    thread_3 = tsinghua('http://news.tsinghua.edu.cn/publish/thunews/9649/{}.html', 640, lock).start()
    thread_4 = tsinghua('http://news.tsinghua.edu.cn/publish/thunews/9650/{}.html', 350, lock).start()
    thread_1.join()
    thread_2.join()
    thread_3.join()
    thread_4.join()
    print('Over !')


if __name__ == "__main__":
    Running()

运行结果：
select * from news; 写入MySQL后查看news表