python爬虫避免重复数据_python爬虫避免重复数据_python 爬虫 内容重复问题

#coding=utf-8importrequestsimportreimportjsonclassNeihan:def__init__(self):self.start_url="http://neihanshequ.com/"self.next_url_temp="http://neihanshequ.com/joke/?is_jso...

# coding=utf-8

import requests

import re

import json

class Neihan:

def __init__(self):

self.start_url = "http://neihanshequ.com/"

self.next_url_temp = "http://neihanshequ.com/joke/?is_json=1&app_name=neihanshequ_web&max_time={}"

self.headers= {"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36"}

def parse_url(self,url):#发送请求，获取htmlstr

print(url)

response = requests.get(url,headers=self.headers)

return response.content.decode()

def get_frist_page_content_list(self,html_str): #提取数据，提取max_time

content_list = re.findall("

.*?

(.*?)",html_str,re.S)

max_time = re.findall("max_time: '(.*?)',",html_str)[0]

return content_list,max_time

def save_content_list(self,content_list): #保存

with open("neihan.txt","a",encoding="utf-8") as f:

for content in content_list:

f.write(content)

f.write("\n")

print("保存成功")

def get_content_list(self,json_str):#提取第二页开始的每一页的数据

dict_ret = json.loads(json_str)

data = dict_ret["data"]["data"]

content_list = [i["group"]["content"] for i in data]

max_time = dict_ret["data"]["max_time"]

has_more = dict_ret["data"]["has_more"]

return content_list,max_time,has_more

def run(self):#实现主要逻辑

#1.start_url

#2.发送请求，获取htmlstr

html_str = self.parse_url(self.start_url)

#3.提取数据，提取max_time

content_list,max_time = self.get_frist_page_content_list(html_str)

#4.保存数据

self.save_content_list(content_list)

#5.构造下一页的url地址

has_more = True #给定一个has_more，假设有第二页

while has_more:

next_url = self.next_url_temp.format(max_time)

#6.发送请求，获取json数据

json_str = self.parse_url(next_url)

#7.提取数据

content_list,max_time,has_more = self.get_content_list(json_str)

#8.保存，进入循环5-8步

self.save_content_list(content_list)

if __name__ == '__main__':

neihan = Neihan()

neihan.run()

#为什么获取到的内容大多数重复，请问如何解决？

展开