这周因为要帮策划做个工具重新用起了python,帮策划做了个筛选修改excel的小工具。然后感觉修改excel的感觉太好了,终于可以不用依赖引擎也能自己做点什么,终于让我感觉我是个程序员,可以按自己的意愿修改系统的东西。然后就想趁着写工具对python的稍微了熟悉了点把自己之前挺想了解的爬虫学习一下,跑到各大网站爬数据,比做UI仔不知道快乐到哪里去了。于是就找了经典的找书和找电影的例子来练习:
以下是代码记录:
# import urllib
# import urllib.request
# import urllib.parse
from urllib import request, parse
import ssl
import requests
import re
import json
from bs4 import BeautifulSoup
import xlwt
import lxml
# response = urllib.request.urlopen("https://www.smzdm.com/")
# result = response.read()
# print(result)
url = "https://www.smzdm.com/?utm_source=baidu&utm_medium=cpc&utm_campaign=0018"
headers = {
# 假装自己是浏览器
# 'User-Agent':' Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'
}
dict = {
'return_url': 'https://www.smzdm.com/',
'user_name': '',
'password': '',
'_post_type': 'ajax',
}
# urllib方式
context = ssl._create_unverified_context()
data = bytes(parse.urlencode(dict), 'utf-8')
req = request.Request(url, data=data, headers=headers, method="POST")
response = request.urlopen(req, context=context)
# print(response.read().decode('utf-8'))
##request 方式
r = requests.get("https://www.smzdm.com/", headers=headers)
# print("uuuu",r.text)
def main(page):
url = 'http://bang.dangdang.com/books/fivestars/01.00.00.00.00.00-all-0-0-1-' + str(page)
html = request_dandan(url)
items = parse_result(html)
for item in items:
write_item_to_file(item)
print(item)
def request_dandan(url):
try:
response = requests.get(url)
if response.status_code == 200:
return response.text
except requests.RequestException:
return None
def parse_result(html):
pattern = re.compile(
'<li>.*?list_num.*?(\d+).</div>.*?<img src="(.*?)".*?class="name".*?title="(.*?)">.*?class="star">.*?class="tuijian">(.*?)</span>.*?class="publisher_info">.*?target="_blank">(.*?)</a>.*?class="biaosheng">.*?<span>(.*?)</span></div>.*?<p><span\sclass="price_n">¥(.*?)</span>.*?</li>',
re.S)
items = re.findall(pattern, html)
for item in items:
yield {
'range': item[0],
'image': item[1],
"title": item[2],
"recommend": item[3],
"author": item[4],
"times": item[5],
"price": item[6],
}
def write_item_to_file(item):
with open("book.txt", "a", encoding="UTF-8")as f:
f.write(json.dumps(item, ensure_ascii=False) + "\n")
f.close()
for i in range(1, 6):
main(i)
def GetDoubanTop250Movie(page):
url = 'https://movie.douban.com/top250?start=' + str(page * 25) + '&filter='
html = request_douban(url)
soup = BeautifulSoup(html, 'lxml')
print(soup)
saveToExcel(soup)
def request_douban(url):
try:
response = requests.get(url, headers=headers)
if response.status_code == 200:
return response.text
except requests.RequestException:
return None
n = 1
def saveToExcel(soup):
list = soup.find(class_="grid_view").find_all('li')
for item in list:
item_name = item.find(class_="title").string
item_img = item.find('a').find('img').get('src')
item_index = item.find(class_="").string
item_score = item.find(class_="rating_num").string
item_author = item.find("p").text
if (item.find(class_='inq') != None):
item_intr = item.find(class_='inq').string
print('爬取电影:' + item_index + ' | ' + item_name + ' | ' + item_score + ' | ' + item_author + ' | ' + item_intr)
global n
sheet.write(n, 0, item_index)
sheet.write(n, 1, item_name)
sheet.write(n, 2, item_intr)
sheet.write(n, 3, item_author)
sheet.write(n, 4, item_img)
n = n + 1
def createExcel():
global book
book = xlwt.Workbook(encoding="utf-8", style_compression=0)
global sheet
sheet = book.add_sheet("豆瓣top250电影", cell_overwrite_ok=True)
sheet.write(0, 0, "排名")
sheet.write(0, 1, "名称")
sheet.write(0, 2, "简介")
sheet.write(0, 3, "作者")
sheet.write(0, 4, "图片")
createExcel()
for i in range(0,11):
GetDoubanTop250Movie(i)
book.save(u'豆瓣最受欢迎的250部电影.xls')
这样就能找到豆瓣top250的电影和当当前100好评的书了,并把数据写入文件。我把程序还打成了exe.csdn上传不了md格式以外的文件,有兴趣的可以用pyinstaller自己打成exe玩一下,还挺有意思的。 python能借助一些库修改文件,获取自己想要的资料,让人真正感受到程序自动化处理的力量,我觉得每个人有空都可以学学,真的挺有意思的。我记得有一次去面试,他问我平时还对什么有兴趣,我说python的爬虫挺有兴趣的,运用自动化收集想要的数据很有意思,能做很多事情,他劝我别分神,有这个精力不如学跟工作相关的东西。我当时就感觉这个人不行,现在感觉更明显了。有意思的事情有兴趣可以多尝试,一点坏处没有,长期陷入眼前工作不能脱身去尝试新东西才是悲剧。