print(‘正在爬取文章:’, article_title[0].strip(‘"’), article_time[0], url)
time.sleep(0.5)
return ‘ok’
视频数据
break_flag_video = []
def shipin(url, max_behot_time=0, csv_name=0, n=0):
max_qingqiu = 20
headers2 = [‘视频发表时间’, ‘标题’, ‘来源’, ‘视频链接’]
first_url = ‘https://www.toutiao.com/c/user/article/?page_type=0&user_id=%s&max_behot_time=%s&count=20&as=%s&cp=%s&_signature=%s’ % (
url.split(‘/’)[-2], max_behot_time, get_as_cp()[‘as’], get_as_cp()[‘cp’],
get_signature(url.split(‘/’)[-2], max_behot_time))
while n < max_qingqiu and not break_flag_video:
try:
res = requests.get(first_url, headers=headers_a, cookies=cookies)
data = json.loads(res.text)
print(data)
max_behot_time = data[‘next’][‘max_behot_time’]
if max_behot_time:
video_list = data[‘data’]
for i in video_list:
try:
start_time = i[‘behot_time’]
video_title = i[‘title’]
video_source = i[‘source’]
detail_url = ‘https://www.ixigua.com/i’ + i[‘item_id’]
resp = requests.get(detail_url, headers=headers())
r = str(random.random())[2:]
url_part = “/video/urls/v/1/toutiao/mp4/{}?r={}”.format(
re.findall(‘“video_id”:“(.*?)”’, resp.text)[0], r)
s = crc32(url_part.encode())
api_url = “https://ib.365yg.com{}&s={}”.format(url_part, s)
resp = requests.get(api_url, headers=headers())
j_resp = resp.json()
video_url = j_resp[‘data’][‘video_list’][‘video_1’][‘main_url’]
video_url = b64decode(video_url.encode()).decode()
print((int(str(time.time()).split(‘.’)[0])-start_time)/86400)
if 30 < (int(str(time.time()).split(‘.’)[0]) - start_time) / 86400 <= 32:
print(‘完成’)
break_flag_video.append(1)
continue
if (int(str(time.time()).split(‘.’)[0]) - start_time) / 86400 > 32:
print(‘完成’)
break_flag_video.append(1)
break
row = {‘视频发表时间’: time.strftime(‘%Y-%m-%d %H:%M:%S’, time.localtime(start_time)),
‘标题’: video_title, ‘来源’: video_source,
‘视频链接’: video_url}
with open(‘/toutiao/’ + str(csv_name) + ‘视频.csv’, ‘a’, newline=‘’, encoding=‘gb18030’)as f:
f_csv = csv.DictWriter(f, headers2)
f_csv.writeheader()
f_csv.writerow(row)
print(‘正在爬取视频:’, video_title, detail_url, video_url)
time.sleep(3)
except Exception as e:
print(e, ‘https://www.ixigua.com/i’ + i[‘item_id’])
shipin(url=url, max_behot_time=max_behot_time, csv_name=csv_name, n=n)
except KeyError:
n += 1
print(‘第’ + str(n) + ‘次请求’, first_url)
time.sleep(3)
if n == max_qingqiu:
print(‘请求超过最大次数’)
break_flag_video.append(1)
except Exception as e:
print(e)
else:
pass
微头条
break_flag_weitoutiao = []
def weitoutiao(url, max_behot_time=0, n=0, csv_name=0):
max_qingqiu = 20
headers3 = [‘微头条发表时间’, ‘来源’, ‘标题’, ‘文章内图片’, ‘微头条内容’]
while n < max_qingqiu and not break_flag_weitoutiao:
try:
first_url = ‘https://www.toutiao.com/api/pc/feed/?category=pc_profile_ugc&utm_source=toutiao&visit_user_id=%s&max_behot_time=%s’ % (
url.split(‘/’)[-2], max_behot_time)
print(first_url)
res = requests.get(first_url, headers=headers_a, cookies=cookies)
data = json.loads(res.text)
print(data)
max_behot_time = data[‘next’][‘max_behot_time’]
weitoutiao_list = data[‘data’]
for i in weitoutiao_list:
try:
detail_url = ‘https://www.toutiao.com/a’ + str(i[‘concern_talk_cell’][‘id’])
print(detail_url)
resp = requests.get(detail_url, headers=headers(), cookies=cookies)
start_time = re.findall(“time: ‘(.*?)’”, resp.text, re.S)
weitoutiao_name = re.findall(“name: ‘(.*?)’”, resp.text, re.S)
weitoutiao_title = re.findall(“title: ‘(.*?)’”, resp.text, re.S)
weitoutiao_images = re.findall(‘images: [“(.*?)”]’,resp.text,re.S)
print(weitoutiao_images)
if weitoutiao_images:
weitoutiao_image = ‘http:’ + weitoutiao_images[0].replace(‘u002F’,‘/’).replace(‘\’,‘’)
print(weitoutiao_image)
else:
weitoutiao_image = ‘此头条内无附件图片’
weitoutiao_content = re.findall(“content: ‘(.*?)’”, resp.text, re.S)
result_time = []
[result_time.append(i) for i in str(start_time[0]).split(’ ‘)[0].replace(’-‘, ‘,’).split(’,')]
print(result_time)
cha = (
datetime.now() - datetime(int(result_time[0]), int(result_time[1]), int(result_time[2]))).days
print(cha)
if cha > 30:
break_flag_weitoutiao.append(1)
print(‘完成’)
break
row = {‘微头条发表时间’: start_time[0], ‘来源’: weitoutiao_name[0],
‘标题’: weitoutiao_title[0].strip(‘"’),‘文章内图片’: weitoutiao_image,
‘微头条内容’: weitoutiao_content[0].strip(‘"’)}
with open(‘/toutiao/’ + str(csv_name) + ‘微头条.csv’, ‘a’, newline=‘’, encoding=‘gb18030’)as f:
f_csv = csv.DictWriter(f, headers3)
f_csv.writeheader()
f_csv.writerow(row)
time.sleep(1)
print(‘正在爬取微头条’, weitoutiao_name[0], start_time[0], detail_url)
except Exception as e:
print(e, ‘https://www.toutiao.com/a’ + str(i[‘concern_talk_cell’][‘id’]))
weitoutiao(url=url, max_behot_time=max_behot_time, csv_name=csv_name, n=n)
except KeyError:
n += 1
print(‘第’ + str(n) + ‘次请求’)
time.sleep(2)
if n == max_qingqiu:
print(‘请求超过最大次数’)
break_flag_weitoutiao.append(1)
else:
pass
except Exception as e:
print(e)
else:
pass
获取需要爬取的网站数据
def csv_read(path):
data = []
with open(path, ‘r’, encoding=‘gb18030’) as f:
reader = csv.reader(f, dialect=‘excel’)
for row in reader:
data.append(row)
return data
启动函数
def main():
for j, i in enumerate(csv_read(‘toutiao-suoyou.csv’)):
data_url = data.get_nowait()
if ‘文章’ in i[3]:
启动抓取文章函数
print(‘当前正在抓取文章第’, j, i[2])
headers1 = [‘发表时间’, ‘标题’, ‘来源’, ‘所有图片’, ‘文章内容’]
with open(‘/toutiao/’ + i[0] + ‘文章.csv’, ‘a’, newline=‘’)as f:
f_csv = csv.DictWriter(f, headers1)
f_csv.writeheader()
break_flag.clear()
wenzhang(url=i[2], csv_name=i[0])
if ‘视频’ in i[3]:
启动爬取视频的函数
print(‘当前正在抓取视频第’, j, i[2])
headers2 = [‘视频发表时间’, ‘标题’, ‘来源’, ‘视频链接’]
with open(‘/toutiao/’ + i[0] + ‘视频.csv’, ‘a’, newline=‘’)as f:
f_csv = csv.DictWriter(f, headers2)
f_csv.writeheader()
break_flag_video.clear()
shipin(url=i[2], csv_name=i[0])
if ‘微头条’ in i[3]:
启动获取微头条的函数
headers3 = [‘微头条发表时间’, ‘来源’, ‘标题’, ‘文章内图片’, ‘微头条内容’]
print(‘当前正在抓取微头条第’, j, i[2])
with open(‘/toutiao/’ + i[0] + ‘微头条.csv’, ‘a’, newline=‘’)as f:
f_csv = csv.DictWriter(f, headers3)
f_csv.writeheader()
break_flag_weitoutiao.clear()
weitoutiao(url=i[2], csv_name=i[0])
多线程启用
def get_all(urlQueue):
while True:
try:
不阻塞的读取队列数据
data_url = urlQueue.get_nowait()
i = urlQueue.qsize()
except Exception as e:
break
print(data_url)
if ‘文章’ in data_url[3]:
# 启动抓取文章函数
print(‘当前正在抓取文章’, data_url[2])
headers1 = [‘发表时间’, ‘标题’, ‘来源’, ‘所有图片’, ‘文章内容’]
with open(‘/toutiao/’ + data_url[0] + ‘文章.csv’, ‘a’, newline=‘’)as f:
f_csv = csv.DictWriter(f, headers1)
f_csv.writeheader()
break_flag.clear()
wenzhang(url=data_url[2], csv_name=data_url[0])
if ‘视频’ in data_url[3]:
启动爬取视频的函数
print(‘当前正在抓取视频’, data_url[2])
headers2 = [‘视频发表时间’, ‘标题’, ‘来源’, ‘视频链接’]
with open(‘/toutiao/’ + data_url[0] + ‘视频.csv’, ‘a’, newline=‘’)as f:
f_csv = csv.DictWriter(f, headers2)
f_csv.writeheader()
break_flag_video.clear()
shipin(url=data_url[2], csv_name=data_url[0])
自我介绍一下,小编13年上海交大毕业,曾经在小公司待过,也去过华为、OPPO等大厂,18年进入阿里一直到现在。
深知大多数Python工程师,想要提升技能,往往是自己摸索成长或者是报班学习,但对于培训机构动则几千的学费,着实压力不小。自己不成体系的自学效果低效又漫长,而且极易碰到天花板技术停滞不前!
因此收集整理了一份《2024年Python开发全套学习资料》,初衷也很简单,就是希望能够帮助到想自学提升又不知道该从何学起的朋友,同时减轻大家的负担。
既有适合小白学习的零基础资料,也有适合3年以上经验的小伙伴深入学习提升的进阶课程,基本涵盖了95%以上Python开发知识点,真正体系化!
由于文件比较大,这里只是将部分目录大纲截图出来,每个节点里面都包含大厂面经、学习笔记、源码讲义、实战项目、讲解视频,并且后续会持续更新
如果你觉得这些内容对你有帮助,可以添加V获取:vip1024c (备注Python)
感谢每一个认真阅读我文章的人,看着粉丝一路的上涨和关注,礼尚往来总是要有的:
① 2000多本Python电子书(主流和经典的书籍应该都有了)
② Python标准库资料(最全中文版)
③ 项目源码(四五十个有趣且经典的练手项目及源码)
④ Python基础入门、爬虫、web开发、大数据分析方面的视频(适合小白学习)
⑤ Python学习路线图(告别不入流的学习)
一个人可以走的很快,但一群人才能走的更远。如果你从事以下工作或对以下感兴趣,欢迎戳这里加入程序员的圈子,让我们一起学习成长!
AI人工智能、Android移动开发、AIGC大模型、C C#、Go语言、Java、Linux运维、云计算、MySQL、PMP、网络安全、Python爬虫、UE5、UI设计、Unity3D、Web前端开发、产品经理、车载开发、大数据、鸿蒙、计算机网络、嵌入式物联网、软件测试、数据结构与算法、音视频开发、Flutter、IOS开发、PHP开发、.NET、安卓逆向、云计算
典的书籍应该都有了)
② Python标准库资料(最全中文版)
③ 项目源码(四五十个有趣且经典的练手项目及源码)
④ Python基础入门、爬虫、web开发、大数据分析方面的视频(适合小白学习)
⑤ Python学习路线图(告别不入流的学习)
一个人可以走的很快,但一群人才能走的更远。如果你从事以下工作或对以下感兴趣,欢迎戳这里加入程序员的圈子,让我们一起学习成长!
AI人工智能、Android移动开发、AIGC大模型、C C#、Go语言、Java、Linux运维、云计算、MySQL、PMP、网络安全、Python爬虫、UE5、UI设计、Unity3D、Web前端开发、产品经理、车载开发、大数据、鸿蒙、计算机网络、嵌入式物联网、软件测试、数据结构与算法、音视频开发、Flutter、IOS开发、PHP开发、.NET、安卓逆向、云计算