思路:BS4爬取电影天堂中最新电影名称及下一级链接中的下载地址,结果保存至csv文件
存在问题及需要完善(恳请各位大神提供帮助):
(1)采集结果中的下载链接存在乱码;
(2)代码需要优化。
# coding=utf-8
import requests
from bs4 import BeautifulSoup
import time
import csv
def getHtml(url):
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0'}
page = requests.get(url, headers=headers)
page.encoding = "gb2312"
html = page.text
#print(html)
soup = BeautifulSoup(html, 'html.parser')
for li in soup.find_all('div', class_="co_content8"):
for url_info in li.find_all('a', class_="ulink"):
moviename= url_info.get_text()
url_1= 'http://www.dytt8.net' + url_info['href']
print(moviename)
print(url_1)
#详情页
req2 = requests.get(url_1, headers=headers)
req2.encoding = 'gb2312'
content2 = req2.content
soup = BeautifulSoup(content2, 'html.parser')
for td in soup.find_all('td', attrs={'style': 'WORD-WRAP: break-word'}):
for url_2 in td.find_all('a'):
url_3 = url_2.string
print(url_3)
item = { # 将获取的结果存储为字典
"name": moviename,
"link": url_1,
"link3": url_3
}
save_result(item) # 每次获取一个结果后,存储一次
item.clear() # 存储后清空字典,为下次存储做准备
#存储
def save_result(item):
#保存在TXT
#with open('result.txt','a ')as f:
#f.write(json.dumps(content) + '\n')
#f.close()
#保存在csv中
with open('dy.csv', 'a', newline='',encoding='utf-8') as csvfile: # 打开一个csv文件,用于存储
fieldnames = ['name', 'link','link3']
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writerow(item)
def main():
with open('dy.csv', 'a', newline='') as csvfile: # 写入表头
writer = csv.writer(csvfile)
writer.writerow(['name', 'link','link3'])
#url = "https://www.dytt8.net/html/gndy/dyzz/index.html"
#getHtml(url)
#翻页1
'''urls = ['https://www.dytt8.net/html/gndy/dyzz/list_23_1.html',
'https://www.dytt8.net/html/gndy/dyzz/list_23_2.html'
]
for url in urls:
getHtml(url)
time.sleep(2)'''
#翻页2
for i in range(1,3):
print('正在访问第'+format(i)+'页')
url ='https://www.dytt8.net/html/gndy/dyzz/list_23_'+ str(i)+'.html'
getHtml(url)
time.sleep(2)
if __name__ == '__main__':
main()
采集结果:(结果中存在乱码)