部分代码分享记一次社会计算作业(四)

最新推荐文章于 2024-06-23 14:58:21 发布

武若叶

最新推荐文章于 2024-06-23 14:58:21 发布

阅读量819

点赞数 1

分类专栏：社会计算第一次作业文章标签： python 爬虫

本文链接：https://blog.csdn.net/qq_45703436/article/details/109565953

版权

社会计算第一次作业专栏收录该内容

4 篇文章 3 订阅

订阅专栏

代码分享

代码仅供交流使用

讲了那么多 , 分享部分我的代码

爬电影总的信息

爬取总的500部电影信息的代码,并保存到
500films.xlsx文件中
( 除了之前讲到的,还用到了json的一点点知识,只有一点点)

import requests
import json
from openpyxl import Workbook
#正常的库引用

wb = Workbook()
ws = wb.active

ws.append(["director","rate","title","actor","url"])
cookie = '' #填你的cookie

headers = {
    "user-agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36",
    "Cookie":cookie
}

#因为这里只请求了一次,所以没有用cookie也是可以的
	
url = "https://movie.douban.com/j/new_search_subjects?sort=U&range=0,10&tags=%E7%94%B5%E5%BD%B1&start=0&limit=500&year_range=2019,2019"

#'''这里的url是之前提到的修改了limit为500的url'''

print('开始爬了')

retval = requests.get(url,headers=headers)
data = json.loads(retval.text)
	
for i in data['data']:
	directors = str(i['directors']).strip("[]").replace('\'','')
	rate = str(i['rate'])
	title = str(i['title'])
	casts = str(i['casts']).strip("[]").replace('\'','')
	url = str(i['url'].replace('"',''))
	ws.append([directors,rate,title,casts,url])
		
#'''非常菜的除括号和引号方法'''

wb.save("500films.xlsx")

在这里插入图片描述

爬评论内容

读取刚刚生成的500films文件,读取里面的url,并且进行相应处理
然后进行爬取十部电影的基本信息和评论内容

# -*- coding: utf-8 -*-  
from bs4 import BeautifulSoup
import requests
import random
import re
import openpyxl 
from openpyxl import Workbook
import time
#'''正常的库引用'''

cookie =  ''  #'''这里填你的cookie'''

FilmName = ''  
temp_column = 1
temp_row = 2  #这两个变量用于指向访问的单元格的行和列

def get_one_url(film_num):	   #读取500films中的url
	global FilmName
	data = openpyxl.load_workbook('./500films.xlsx')
	table = data.get_sheet_by_name('Sheet')
	url = table.cell(film_num+1,5).value+"comments?start="
	FilmName = table.cell(film_num+1,3).value
	return url 

def dispose_url(url,comment_num):  #处理和调整url模拟翻页
	url = url + str(comment_num) + "&limit=20&status=P&sort=new_score"
	return url

def get_html(url):     #获取我们要的html文件
	headers = {
    "user-agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36",
	"Cookie":cookie
	}
	
	response = requests.get(url,headers = headers)
	time.sleep(random.randint(10,20)/10)   
	 #'''这一步让每次爬取时都能随机休眠0.5-1s,太规整的休眠容易被识别,没有休眠也容易导致请求太频繁被豆瓣封ip'''
	#print("here is"+response.text)
	return response

def get_soup(response):    #beautifulsoup化
	demo = BeautifulSoup(response,"html.parser")
	return demo

def get_infor(demo):  #读取总的信息
	infor = demo.find_all("span",class_="attrs")
	infor = infor[0].find_all("p")
	dele = re.compile('\s')
	global temp_column,temp_row
	temp_column = 2;
	for i in infor:
		#print(temp_row,temp_column)
		temp = dele.sub('',i.text)[3:]
		#print(temp)
		ws.cell(row = temp_row,column = temp_column).value = temp
		temp_column += 1
#'''对html页面的读取处理和信息的存储'''
	
def get_comments(demo):    #'''获取评论'''
	a = demo.find_all("span",class_="short")
	global temp_column,temp_row
	for i in a:
		#print(i.text)
		ws.cell(row = temp_row,column = temp_column).value = i.text
		temp_column += 1
	temp_row += 1

def get_more_comments(url1):   #'''循环获取评论'''
	global temp_column,temp_row
	for i in range(0,26):
		temp_column = 8 
		url = dispose_url(url1,i*20)
		print("=========="+url+"==========")
		response = get_html(url)
		demo = get_soup(response.text)
		get_comments(demo)
	
#f = open("douban.html","r",encoding='UTF-8')
#response = f.read()   
wb = Workbook()
ws = wb.active
ws.append(["name","director","actors","type","area","time","release time","comments"])


for film_num in range(1,11):
	url1 = get_one_url(film_num)
	url = dispose_url(url1,0)
	response = get_html(url)
	demo = get_soup(response.text)
	get_infor(demo)
	ws.cell(temp_row,1).value = str(FilmName)  
	get_more_comments(url1)


wb.save('film_comments.xlsx')