python爬虫避免重复数据_python 爬虫 内容重复问题

#coding=utf-8importrequestsimportreimportjsonclassNeihan:def__init__(self):self.start_url="http://neihanshequ.com/"self.next_url_temp="http://neihanshequ.com/joke/?is_jso...

# coding=utf-8

import requests

import re

import json

class Neihan:

def __init__(self):

self.start_url = "http://neihanshequ.com/"

self.next_url_temp = "http://neihanshequ.com/joke/?is_json=1&app_name=neihanshequ_web&max_time={}"

self.headers= {"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36"}

def parse_url(self,url):#发送请求，获取htmlstr

print(url)

response = requests.get(url,headers=self.headers)

return response.content.decode()

def get_frist_page_content_list(self,html_s