和xun基金排名爬取

大约到200多页的时候，出现了反爬

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
import time
from bs4 import BeautifulSoup
import pymongo
import datetime

class heshunSpider(object):
    client =pymongo.MongoClient(host="127.0.0.1", port=27017)
    db = client.hexun.hx
    def __init__(self):
        self.driver = webdriver.Firefox()

    def __del__(self):
        self.driver.close()

    def run(self):
        self.driver.maximize_window()
        self.driver.get("http://jingzhi.funds.hexun.com/fund4th/shouyi.aspx")
        time.sleep(1)
        page = 0
        while page<399:
            time.sleep(6)
            html = self.driver.page_source
            soup = BeautifulSoup(html,'lxml')
            time.sleep(1)
            tr_list = soup.select('.tab01>table>tbody>tr')
            for tr in tr_list:
                item = {}
                item['xuhao'] = tr.find_all("td")[1].string
                item['code'] = tr.find_all("td")[2].find("a").string
                item['name'] = tr.find_all("td")[3].find("a").string
                item['date'] = tr.find_all("td")[4].get_text()
                item['onem'] = tr.find_all("td")[5].string
                item['threem'] = tr.find_all("td")[6].string
                item['sixm'] = tr.find_all("td")[7].string
                item['oney'] = tr.find_all("td")[8].string
                item['twoy'] = tr.find_all("td")[9].string
                item['threey'] = tr.find_all("td")[10].string
                item['todayy'] = tr.find_all("td")[11].string
                item['create'] = tr.find_all("td")[12].string
                item['rate'] = tr.find_all("td")[13].string
                self.db.insert_one(item)
                # print(item)
            self.driver.execute_script("window.scrollTo(0,document.body.scrollHeight);")
            page = page+1
            self.driver.find_element_by_class_name('next').click()



if __name__ == "__main__":
    spider = heshunSpider()
    spider.run()

存入mongodb

注：我是个大菜鸟，恳求大神指正，让我进步！