以安徽日报公众号为例
原理上是基于微信公众号订阅号的机制
测试1000以下不会封号,但是一次性爬取超过1100多就会封号一段时间
# -*- coding: utf-8 -*-
import requests
import time
import xlwt
import xlrd
import re
import os
import threading
s = requests.session()
s.keep_alive = False
class AnhuiDailySpider():
def __init__(self, KeyWords=["疫情"]):
'''网页请求的两个重要参数'''
self.cookies = "这里填写的是微信订阅号网页请求的cookies"
self.DownloadHeaders = {
"Cookie": self.cookies,
"User-Agent": "Mozilla/5.0 (Linux; Android 10; STK-AL00 Build/HUAWEISTK-AL00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/78.0.3904.62 XWEB/2759 MMWEBSDK/201201 Mobile Safari/537.36 MMWEBID/1378 \
MicroMessenger/8.0.1.1841(0x280001B6) Process/toolsmp WeChat/arm64 Weixin NetType/WIFI Language/zh_CN ABI/arm64"
}
'''新闻计数器'''
self.NewsCount = 0
'''新闻标题-新闻链接-新闻时间保存的位置'''
self.workbook = xlwt.Workbook(encoding='utf-8')
self.Worksheet = self.workbook.add_sheet('AuhuiDaily_articles')
self.NewsPath = "AnhuiDaily.xls"
'''新闻内容保存的位置'''
self.NewsContentPath = "./Download/"
'''获取新闻内容请求的相关参数'''
self.GetContentHeaders = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36"
}
'''爬取新闻链接的网址'''
self.wechat_url = "https://mp.weixin.qq.com/cgi-bin/appmsg"
'''新闻计数器'''
self.NewsCount = 0
'''方便断点续传的参数'''
self.startPages = self.downloadPages()
'''关键词下载选项'''
self.KeyWords = KeyWords
'''新闻中需要清洗的内容'''
self.needClearWords = ['<!-- 注意:这个文件是一个公共文件,被很多地方引用,改动需要注意其他类型页面是否有受影响 -->', '安徽日报']
self.createSavePath()
def createSavePath(self):
if not os.path.exists(self.NewsContentPath):
os.mkdir(self.NewsContentPath)
def downloadPages(self):
AnhuiDaily = xlrd.open_workbook("downloadPages.xls")
AnhuiDailyArticles = AnhuiDaily.sheets()[0]
downloadPages = AnhuiDailyArticles.cell(0, 0).value
return int(downloadPages)
'''把时间戳转换为具体的事件点'''
def timeStampTrans(timeStamp):
timeArray = time.localtime(timeStamp)
otherStyleTime = time.strftime("%y-%m-%d %H:%M:%S", timeArray)
return otherStyleTime
"""
需要提交的data
以下个别字段是否一定需要还未验证。
注意修改yourtoken,number
number表示从第number页开始爬取,为5的倍数,从0开始。如0、5、10……
token可以使用Chrome自带的工具进行获取
fakeid是公众号独一无二的一个id,等同于后面的__biz
"""
def downloadNewsUrls(self):
'''检查下载了多少篇文章,断点续传'''
for newsPage in range(self.startPages, 316):
print("正在下载"+str(newsPage)+"页")
data = {
"token": 1130076224,
"lang": "zh_CN",
"f": "json",
"ajax": "1",
"action": "list_ex",
"begin": str(newsPage),
"count": "5",
"query": "",
"fakeid": "MzA5NTc5ODQwMA==",
"type": "9",
}
'''使用get方法获取'''
content_json = requests.get(self.wechat_url, headers=self.DownloadHeaders, params=data).json()
'''返回了一个json,里面是每一页的数据'''
beforeCount = self.NewsCount
try:
for item in content_json["app_msg_list"]:
NewsTitle = item['title']
Newslink = item['link']
NewsTime = self.timeStampTrans(item['create_time'])
SingleNews = [NewsTitle, Newslink, NewsTime]
for i in range(0, 3):
self.Worksheet.write(self.NewsCount, i, SingleNews[i])
self.NewsCount += 1
print("第"+str(newsPage)+"页,"+"有"+str(self.NewsCount-beforeCount)+"篇新闻,"+"共"+str(self.NewsCount)+"篇新闻")
time.sleep(3)
self.workbook.save(self.savePath)
except KeyError:
if newsPage != self.downloadPages():
Pages = xlwt.Workbook(encoding='utf-8')
PageSheet = Pages.add_sheet('downloadPages')
PageSheet.write(0, 0, str(newsPage))
Pages.save("downloadPages.xls")
else:
print("封号中")
break
def downLoadArticles(self):
for newsPage in range(0, 316):
print("正在下载"+str(newsPage)+"页")
data = {
"token": 271655701,
"lang": "zh_CN",
"f": "json",
"ajax": "1",
"action": "list_ex",
"begin": str(newsPage),
"count": "5",
"query": "",
"fakeid": 'MzA5NTc5ODQwMA==',
"type": "9",
}
# 使用get方法进行提交
content_json = requests.get(self.url, headers=self.DownloadHeaders, params=data).json()
# 返回了一个json,里面是每一页的数据
for item in content_json["app_msg_list"]:
NewsTitle = item['title']
Newslink = item['link']
NewsTime = self.StampTrans(item['create_time'])
SingleNews = [NewsTitle, Newslink, NewsTime]
for i in range(0, 3):
self.Worksheet.write(self.NewsCount+1, i, SingleNews[i])
self.NewsCount += 1
print(self.NewsCount)
time.sleep(10)
self.workbook.save(self.NewsPath)
print("下载完成,一共是 "+str(self.NewsCount)+" 篇新闻")
'''进行数据清洗'''
def dataClear(self, data):
for word in self.needClearWords:
data = data.replace(word, '')
return data
def collectNewsContent(self, threads):
totalPages = len(self.readUrl("AnhuiDaily_Part1.xls"))
print("共有 "+str(totalPages) + " 条新闻需要下载")
for index in range(1, 100):
if totalPages % index == 0:
thread_num = int(totalPages/index)
print("开启了 "+str(thread_num)+" 个线程")
each = totalPages/thread_num
i = -1
for i in range(int(thread_num)-1):
startPage, endPage = int(i*each), int((i+1)*each)
t = threading.Thread(target=ScrapyInfo, args=[self.DownloadHeaders, startPage, endPage])
t.start()
threads.append(t)
time.sleep(1)
t = threading.Thread(target=ScrapyInfo, args=(self.DownloadHeaders, startPage, endPage))
t.start()
threads.append(t)
for thread in threads:
thread.join()
def readUrl(self, path):
ArticlesUrls = []
AnhuiDaily = xlrd.open_workbook(path)
AnhuiDailyArticles = AnhuiDaily.sheets()[0]
for newsIndex in range(AnhuiDailyArticles.nrows):
if AnhuiDailyArticles.cell(newsIndex, 1) is None:
break
ArticlesUrls.append(AnhuiDailyArticles.cell(newsIndex, 1).value)
return ArticlesUrls
def readTitle(self, path):
ArticlesTitle = []
AnhuiDaily = xlrd.open_workbook(path)
AnhuiDailyArticles = AnhuiDaily.sheets()[0]
for newsIndex in range(AnhuiDailyArticles.nrows):
if AnhuiDailyArticles.cell(newsIndex, 0) is None:
break
ArticlesTitle.append(AnhuiDailyArticles.cell(newsIndex, 0).value)
return ArticlesTitle
def ScrapyInfo(headers, startPage, endPage):
AnhuiDailyDemo = AnhuiDailySpider()
Urls = AnhuiDailyDemo.readUrl(path="AnhuiDaily_Part1.xls")
Titles = AnhuiDailyDemo.readTitle(path="AnhuiDaily_Part1.xls")
for urlIndex in range(startPage, endPage):
try:
res = requests.get(url=Urls[urlIndex], headers=headers, timeout=30)
except Exception as e:
print("********产生了一个异常***********")
print(e+"异常的链接: "+Urls[urlIndex])
continue
print(str(urlIndex)+" "+Titles[urlIndex])
res.encoding = 'utf-8'
NewsContent = AnhuiDailyDemo.dataClear(res.text)
regex = re.compile(r'[\u4e00-\u9fa5]+')
chineseResult = regex.findall(NewsContent)
NewsContentString = ""
for String in chineseResult:
NewsContentString = NewsContentString+String
with open(AnhuiDailyDemo.NewsContentPath+str(urlIndex)+".txt", "a+") as f:
f.write(NewsContentString)
if __name__ == "__main__":
'''放置线程的列表'''
threads = []
'''开启的线程的数量'''
thread_nums = 400
os.system("cls")
KeyWords = ["疫情"]
AnhuiDailyDemo = AnhuiDailySpider(KeyWords=KeyWords)
AnhuiDailyDemo.collectNewsContent(threads)