# -*- coding: utf-8 -*-
'''
@author: Yalei Meng E-mail: yaleimeng@sina.com
@license: (C) Copyright 2017, HUST Corporation Limited.
@desc:获取新浪热门微博内容。保存为txt文件
@DateTime: Created on 2017/10/3,at 15:48 '''
from bs4 import BeautifulSoup as bs
import requests as rq
import time
import random
import json
import csv
cookie = ?
def request_page(Page):
head = {'Accept': 'application / json, text / plain, * / *',
'Cookie':cookie, #将登陆后自己的cookie粘贴在这里即可。
'Referer':'https://m.weibo.cn/p/index?containerid=102803',
'User-Agent':'Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) '
'Version/9.0 Mobile/13B143 Safari/601.1',
'X-Requested-With':'XMLHttpRequest' }
r = rq.get(Page,headers = head,timeout = 5)
return r.json()
#11353296
def get_text_from(page):
js = request_page(page)
cards = js.get('data').get('cards')
print(len(cards))
t_list = []
if not cards:
return t_list
for a in range(len(js['data']['cards'])):
try:
a_txt = js['data']['cards'][a]['mblog']['text'].split('<')[0]
# pub_time = js['cards'][a]['mblog']['created_at']
# a_pic = js['cards'][a]['mblog'].get('original_pic')
t_list.append(a_txt)
except Exception as ex:
print(ex)
return t_list
#延迟刷新访问,可能存在微博内容重复的问题。需要注意去重。
url_list = ['https://m.weibo.cn/api/container/getIndex?containerid=102803&since_id={}'.format(str(i))
for i in range(0,1000)]
#url_list=['https://weibo.cn/search/mblog?hideSearchFrame=&keyword=%E5%8F%B0%E9%A3%8E%E5%B1%B1%E7%AB%B9&page={}'.format(str(i))
# for i in range(0,100)]
url="https://weibo.cn/search/?tf=5_012"
#如果要批量访问,循环访问列表的url即可。注意访问间隔不能太短。小心被封号
for i, url in enumerate(url_list,1):
mylist = get_text_from(url)
print('当前第%d页'%i,mylist)
with open('E:/weibo1010.txt', 'a',encoding= 'utf-8')as f:
for my in mylist:
f.write(my)
f.write('\n')
time.sleep(random.uniform(1.2,3.0))
print('恭喜,程序运行完毕!')
引用了github 某位程序员的代码,等修改完成后(抓取某热点的话题)再整理上传。