- 解决列表页可配置URL提取、自动翻页、自动结束的问题,功能强大,代码紧凑,生产环境可用
- 爬虫URL生产者
- 爬虫URL消费者可以参考
- 支持GET、POST提取URL列表
- 翻页结束条件支持总页数、第几页、下一页为空三种
- 支持指定起始页
- 支持提取子页(文章评论、论坛跟帖,先爬id再拼接URL等)URL提取
- 支持同一域名,多版块URL提取简写【a,b,c】实现
- 支持不同意URL优先提取功能,配置level
- sub、{}、=>、==、【】等自定义占位符实现
import pandas as pd
import requests
from scrapy.selector import Selector
import re
from urllib import parse
import json
import jsonpath
import math
import urllib3
import copy
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
class Goto:
def __init__(self):
self.over_max_num = 100
def first_request(self, row, urls, total_num=-1):
start_url = row.get('start_url')
page_url = row.get('page_url') if not self.isnan(row.get('page_url')) else ''
page_extract = ''
request_type = row.get('request_type') if not self.isnan(row.get('request_type')) else 'GET'
pre_re = row['pre_re']
detail = row['detail']
total_count_path = row['total_count']
total_page_path = row['total_page']
over = row['over']
if '{' in page_url:
page_extract = page_url[page_url.find('{') + 1:page_url.find('}')]
if '//' in page_extract or '$' in page_extract:
if total_num == -1:
page_url = ''
else:
# //div=>123 拆解示例 xpath翻页
page_url = page_url.replace(page_url[page_url.find('{') + 1:page_url.find('=>') + 2], '')
row['page_url'] = row['page_url'].replace(
row['page_url'][row['page_url'].find('{') + 1:row['page_url'].find('}')],
page_extract.split('=>')[0]
)
page_url = page_url.replace('{', '').replace('}', '')
try:
res = self.__get_response(start_url=start_url, page_url=page_url, request_type=request_type)
except:
return []
if self.isnan(detail):
urls.append(parse.urljoin(start_url, page_url))
else:
detail_path = detail
pre_re_extract = res.text
if not self.isnan(pre_re):
pre_re_extract = self.__extract_filer(res.text, pre_re)
if len(pre_re_extract) > 0:
pre_re_extract = pre_re_extract[0]
detail_url = '' # 自定义详情页url拼接时候使用
if '{' in detail_path and '}' in detail_path:
detail_url = detail_path
detail_path = detail_url[detail_url.find('{') + 1:detail_url.find('}')]
detail_list = self.__extract_filer(text=pre_re_extract, path=detail_path)
for _, v in enumerate(detail_list):
if detail_url == '':
urls.append(parse.urljoin(start_url, v))
else:
urls.append(detail_url.replace(detail_url[detail_url.find('{'):detail_url.find('}') + 1], str(v)))
# 返回单页URL
if len(page_url) == 0 and ('//' not in page_extract or '$' not in page_extract):
return urls
# 结束计算
if self.isnan(over):
# 第一次执行
if total_num == -1:
if not self.isnan(total_count_path):
arr = self.__extract_filer(text=res.text, path=total_count_path)
total_num = max(math.ceil(int(arr[0]) / len(detail_list)), 1) if len(arr) > 0 else 0
elif not self.isnan(total_page_path):
arr = self.__extract_filer(text=res.text, path=total_page_path)
total_num = int(arr[0]) if len(arr) > 0 else 0
else:
curr_value, over_value = over.split('==')[0], over.split('==')[1]
curr_value = self.__extract_filer(res.text, curr_value)
curr_value = curr_value[0] if len(curr_value) > 0 else False
if total_num == -1:
total_num = self.over_max_num
else:
if not curr_value or curr_value == over_value:
total_num = 0
total_num -= 1
if total_num >= 1:
self.__next_request(row=row, res=res, request_type=request_type, urls=urls, total_num=total_num)
# 回滚一条无效url, 递归最外层self.over_max_num - 1
urls = urls[0:-1] if total_num == self.over_max_num - 1 else urls
return urls
# post请求数据处理
def __get_post_data(self, page_url):
data = {}
page_url = page_url.replace('\t', '').replace('\r', '').replace('\n', '')
arr = page_url.split(',')
for i, v in enumerate(arr):
v = v.replace("\n", '')
v = v.strip()
key = v.split(':')[0].strip()
value = v.split(':')[1].strip()
data[key] = value
return data
# 详情页之后接口调用支持url,url_list,html
def sub_row(self, row, master_urls):
sub_url_list = []
if isinstance(master_urls, str):
master_urls = [master_urls]
for _, v in enumerate(master_urls):
page_url = row['page_url']
new_page_url = self.__get_page_param(page_url=page_url, url=v)
if not new_page_url:
print(f'sub提取url __get_page_param提取{v}参数错误')
continue
row['page_url'] = new_page_url
sub_url_list += self.first_request(row=row, urls=[])
return sub_url_list
# 遍历起始url
def for_url(self, row, keywords):
page_url = row.get('page_url') if not self.isnan(row.get('page_url')) else ''
row_list = []
if page_url:
if isinstance(keywords, str):
keywords = keywords.replace('\n', '').replace('\r', '').replace('\t', '').strip().split(',')
for _, v in enumerate(keywords):
new_row_dict = copy.deepcopy(row)
new_page_url = page_url.replace(page_url[page_url.find('【'):page_url.find('】') + 1], str(v))
if not new_page_url:
print(f'for_url提取关键词 {v}参数错误')
continue
new_row_dict['page_url'] = new_page_url
row_list.append(new_row_dict)
else:
row_list.append(row)
return row_list
# 获取page_url起始【】里的内容
def get_for_url_keyword(self, row):
page_url = row.get('page_url') if not self.isnan(row.get('page_url')) else ''
keyword = ''
if '【' in page_url and '】' in page_url:
keyword = str(page_url[page_url.find('【') + 1:page_url.find('】')]).strip()
return keyword
# 下一页调用
def __next_request(self, row, res, request_type, urls, total_num):
page_url = row['page_url']
s0 = page_url.find('{')
s1 = page_url.find('}')
page_extract = page_url[page_url.find('{') + 1:page_url.find('}')]
start_flag = self.__extract_filer(res.text, page_extract)[0]
if '//' in page_extract:
start_flag = f'{page_extract}=>{start_flag}'
else:
start_flag = str(int(start_flag) + 1)
next_page = page_url[:s0 + 1] + start_flag + page_url[s1:]
if request_type == 'POST':
arr = next_page.split(',')
for k, v in enumerate(arr):
if '=//' in v:
v = v.replace("\n", '')
v = v.strip()
key = v.split('=//')[0]
value = v.split('=//')[-1]
value = f"//{value.split(':')[0]}"
arr[k] = f"{key}={value}:{self.__extract_filer(res.text, value)[0]}"
next_page = ','.join(arr)
# print(f'next_page:{next_page}')
row['page_url'] = next_page
self.first_request(row=row, urls=urls, total_num=total_num)
# 请求一个URL,返回selector
def __get_response(self, start_url, page_url=None, request_type='GET'):
url = start_url
header = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; rv:26.0) Gecko/20100101 Firefox/26.0'}
if request_type == 'GET':
if page_url:
url = parse.urljoin(start_url, page_url)
# print(f'url: {url}')
res = requests.get(url, headers=header, verify=False, timeout=30)
elif request_type == 'POST':
data = self.__get_post_data(page_url=page_url)
# print('data', data)
res = requests.post(url, data=data, verify=False, timeout=30)
res.encoding = 'utf-8'
return res
# path提取加正则过滤,其中提取支持json、xpath、整数、正则
def __extract_filer(self, text, path):
grep = None
value = []
if isinstance(path, str):
if '|' in path:
grep = path.split('|')[-1]
path = path.split('|')[0]
if '$' in path[0]:
obj = json.loads(text)
temp = jsonpath.jsonpath(obj, path)
value = jsonpath.jsonpath(obj, path) if temp else []
elif '//' in path[:2]:
selector = Selector(text=text)
value = selector.xpath(path).extract()
else:
value = self.__filter(obj=text, regular=path)
if grep:
if len(value) > 0:
value = self.__filter(obj=value, regular=grep)
else:
value = self.__filter(obj=text, regular=grep)
elif isinstance(path, int) or isinstance(path, float):
value = [path]
return value
# 对get请求参数计算
def __get_page_param(self, page_url, url):
if '&' in page_url:
params = str(page_url).split('&')
elif '?' in page_url:
params = str(page_url).split('?')
for i, param in enumerate(params):
if '=>' in param:
k_v = str(param).split('=>')
value = self.__extract_filer(url, path=k_v[-1])
if len(value) == 0:
return False
k_v[-1] = value[0]
param = '='.join(k_v)
params[i] = param
page_url = '&'.join(params)
return page_url
# 正则函数
def __filter(self, obj, regular):
text = obj
if not isinstance(obj, str):
text = ''.join(obj)
pattern = re.compile(regular)
return pattern.findall(text)
# 空值判断
def isnan(self, num):
return num != num
if __name__ == '__main__':
import time
t = time.time()
excel_path = '../../data/goto.xlsx'
df = pd.read_excel(excel_path)
goto = Goto()
# 0 暂停信源
df = df[df['level'] != 0]
df = df.sort_values('level', axis=0, ascending=False)
keyword_dict = {}
words_df = pd.read_csv(f'../../data/goto_keywords.csv')
keyword_dict[words_df['words'].values.tolist()[0]] = words_df['words'].values.tolist()
keyword_dict[words_df['code_utf-8'].values.tolist()[0]] = words_df['code_utf-8'].values.tolist()
keyword_dict[words_df['code_gbk'].values.tolist()[0]] = words_df['code_gbk'].values.tolist()
test_http = 'http://info.xcar.com.cn/202007/news_2049855_1.html'
domain_valid_dict = {'count': 0, 'urls_len': 0}
for i, row in df.iterrows():
if 'sub' not in row['domain_flag']:
row_dict = row.to_dict()
words = goto.get_for_url_keyword(row_dict).upper()
# 默认都是用英文字母大写转换
words = keyword_dict[words] if keyword_dict.get(words) else words
row_list = goto.for_url(row_dict, words)
for _, v in enumerate(row_list):
urls = list(set(goto.first_request(row=v, urls=[])))
print(f'第{i}个信源 {v["source"]} {len(urls)} {urls}')
domain_valid_dict['count'] += 1
domain_valid_dict['urls_len'] += len(urls)
else:
urls = goto.sub_row(row=row.to_dict(), master_urls=test_http)
print(f'第{i}个信源 level: {row["level"]} {row["source"]} {len(urls)} {urls}')
domain_valid_dict['count'] += 1
domain_valid_dict['urls_len'] += len(urls)
print(f'有效信源:{domain_valid_dict["count"]}个,'
f' 共提取url:{domain_valid_dict["urls_len"]}个, '
f'总耗时:{round(time.time() - t, 2)}秒, '
f'平均耗时:{round(domain_valid_dict["urls_len"]/(time.time() - t), 2)}条/秒')
goto URL配置文件截图