Python3爬取网页数据，导入数据库并去重 设置间隔时间爬取。

#用来操作数据库的类

	import pymysql
	class MySQLCommand(object):

    def __init__(self):
        self.host = 'localhost'
        self.port = 3306  # 端口号
        self.user = 'root'  # 用户名
        self.password = "root"  # 密码
        self.db = "home"  # 库
        self.table = "home_list1"  # 表

    # 链接数据库
    def connectMysql(self):
        try:
            self.conn = pymysql.connect(host=self.host, port=self.port, user=self.user,
                                        passwd=self.password, db=self.db, charset='utf8')
            self.cursor = self.conn.cursor()
        except:
            print('connect mysql error.')

    # 插入数据，插入之前先查询是否存在，如果存在就不再插入
    def insertData(self, my_dict):
        #table = "home_list"  # 要操作的表格
        # 注意，这里查询的sql语句url=' %s '中%s的前后要有空格
        sqlExit = "SELECT id FROM home_list1  WHERE lj = ' %s '" % (my_dict['lj'])
        res = self.cursor.execute(sqlExit)
        if res:  # res为查询到的数据条数如果大于0就代表数据已经存在
            print("数据已存在", res)
            return 0
        # 数据不存在才执行下面的插入操作
        try:
            cols = ', '.join(my_dict.keys())#用，分割
            values = '"," '.join(my_dict.values())
            sql = "INSERT INTO home_list1 (%s) VALUES (%s)" % (cols, '"' + values + '"')
            #拼装后的sql如下
            # INSERT INTO home_list (id, name, time, lj) VALUES ("https://img.huxiucdn.com.jpg"," https://www.huxiu.com90.html"," 12"," ")
            try:
                result = self.cursor.execute(sql)
                insert_id = self.conn.insert_id()  # 插入成功后返回的id
                self.conn.commit()
                # 判断是否执行成功
                if result:
                    print("插入成功", insert_id)
                    return insert_id + 1
            except pymysql.Error as e:
                # 发生错误时回滚
                self.conn.rollback()
                # 主键唯一，无法插入
                if "key 'PRIMARY'" in e.args[1]:
                    print("数据已存在，未插入数据")
                else:
                    print("插入数据失败，原因 %d: %s" % (e.args[0], e.args[1]))
        except pymysql.Error as e:
            print("数据库错误，原因%d: %s" % (e.args[0], e.args[1]))

    # 查询最后一条数据的id值
    def getLastId(self):
        sql = "SELECT max(id) FROM " + self.table
        try:
            self.cursor.execute(sql)
            row = self.cursor.fetchone()  # 获取查询到的第一条数据
            if row[0]:
                return row[0]  # 返回最后一条数据的id
            else:
                return 0  # 如果表格为空就返回0
        except:
            print(sql + ' execute failed.')

    def closeMysql(self):
        self.cursor.close()
        self.conn.close()  # 创建数据库操作类的实例

	# -*- coding: utf-8 -*-
	"""
	Created on Wed Jan 16 13:43:37 2019
	
	@author: Administrator
	"""
	
	import urllib.request
	from bs4 import BeautifulSoup
	from pcsql import MySQLCommand
	from threading import Timer
	import time
	
	def get_html(url):
		headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WDW64; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'}  
		req = urllib.request.Request(url, headers=headers) 
		page = urllib.request.urlopen(req)
		html = page.read().decode('UTF-8')
		return html
	
	
	def infos_paser():
		url = 'http://www.mnr.gov.cn/dt/hy/index.html'
		htmlCode = get_html(url)
		soup = BeautifulSoup(htmlCode,'html.parser')
		ul_info_list = soup.find_all('ul', attrs={'class': 'ky_open_list'}, limit=1)
		li_info_list=ul_info_list[0].find_all('li')
	
		for item in li_info_list:
			analyzeInfo(item)
	
	def analyzeInfo(item):
		result = {}
		a_title = item.find_all('a',limit=1)
		if a_title:
			dataCount = int(mysqlCommand.getLastId()) + 1
			result['id'] = str(dataCount)
			#公告标题
			result['name'] = a_title[0].get_text().strip()
	    	#公告链接
			link = 'http://www.mnr.gov.cn/dt/hy'
			result['lj'] = link + a_title[0]['href'].strip('.')
			span_time = item.find_all('span',limit=1)
			result['time'] = span_time[0].get_text().strip()
			try:
	            # 插入数据，如果已经存在就不在重复插入
				res = mysqlCommand.insertData(result)
				if res:
					dataCount=res
			except Exception as e:
				print("插入数据失败", str(e))#输出插入失败的报错语句
	
	def rund():
	    mysqlCommand.connectMysql()
	
	    infos_paser()
	
	    mysqlCommand.closeMysql()  # 最后一定要要把数据关闭
	    t = Timer(10, rund)#10秒爬取一次
	    t.start()
	
	
	#---------------------------------------------------------------
	
	if __name__ == "__main__":
	    # 连接数据库
	    mysqlCommand = MySQLCommand()
	    rund()