爬取起点小说
import requests
from bs4 import BeautifulSoup
import bs4
path="3.txt" # 这里目录也可以改成绝对目录
fw=open(path,'w',encoding='utf-8') # open函数要加上编码为'utf-8'不然会乱码
url = "https://book.qidian.com/info/1015323848#Catalog"
r=requests.get(url,timeout=30)
r.raise_for_status() # 状态码的检测
r.encoding=r.apparent_encoding
soup = BeautifulSoup(r.text, 'html.parser')
for li in soup.find_all('ul')[3].children: # 标签树下行遍历,试了一下发现也可以把.children去掉
if isinstance(li, bs4.element.Tag): # 如果遍历的lis是一个bs4的tag标签,就处理它
chapter = li.a
# print(chapter)
chapter_url = 'https:%s' % chapter.get('href')
#chapter_title=chapter.contents # ->['第一节 龙城']
chapter_title=chapter.text # ->第一节 龙城
# print(chapter_title)
# print(chapter_url)
chapter_response=requests.get(chapter_url)
chapter_response.encoding=chapter_response.apparent_encoding
chapter_html=chapter_response.text
chapter_soup=BeautifulSoup(chapter_html,'html.parser')
chapter_tag=chapter_soup.find('div',class_='read-content j_readContent')
fw.write(chapter_title)
fw.write(chapter_tag.text)
fw.close()
这里写的很乱,没有整理,只是为了实现这个功能,最后成功把爬取的数据写入了文件,如图
爬取一张图片
https://wx4.sinaimg.cn/mw690/8d05b653ly1g4n0elm7axj20j60b475b.jpg
import requests
path='D://abc.jpg' # 保存在D盘下,以abc.jpg命名
url='https://wx4.sinaimg.cn/mw690/8d05b653ly1g4n0elm7axj20j60b475b.jpg'
# 以原文件名命名
# roo='D://'
# path=root+url.split('/')[-1]
r=requests.get(url)
with open(path,'wb') as f:
f.write(r.content)
首先打开一个文件,将url返回的内容写到这个路径中,r.content返回的是二进制形式;
爬取天猫Ipad商品信息
import requests
import re
import json
def getHtml(url) :
try:
reponse = requests.get(url)
if reponse.status_code != 200:
return None
else :
return reponse.text
except:
return None
def getInf(html):
pattern =re.compile('<div class="product.*?<a href="(.*?)".*?</b>(.*?)</em>.*?title="(.*?)".*?<a.*?>(.*?)</a>.*?</div>', re.S)
items = re.findall(pattern, html)
for item in items:
yield {
'href':item[0],
'price':item[1],
'title':item[2],
'shop':item[3]
}
def writeFile(content):
with open('3.txt', 'a', encoding = 'utf-8') as f:
f.write(json.dumps(content, ensure_ascii = False) + '\n')
f.close()
def main():
url ='https://list.tmall.com/search_product.htm?q=ipad&type=p&vmarket=&spm=875.7931836%2FB.a2227oh.d100&from=mallfp..pc_1_searchbutton'
html = getHtml(url)
for item in getInf(html):
print(item)
writeFile(item)
if __name__ == '__main__':
main()
爬取2020USTC研究生拟录取名单
下面的代码只给出了一部分关键的部分的,需要提前获取官网上拟录取公告的html,之后保存为文本文件;
import requests
import re
import json
import pandas as pd
import sys
import os
# print('当前 Python 解释器路径:')
# print(sys.executable)
def getInf(html):
pattern = re.compile('<tr style=\'height:14.15pt.*?<td width=.*?<p class=MsoNormal.*?<span.*?>(.*?)</span>.*?\
<td width=.*?<p class=MsoNormal.*?<span.*?>(.*?)</span>.*?\
<td width=.*?<p class=MsoNormal.*?<span.*?>(.*?)</span>.*?\
<td width=.*?<p class=MsoNormal.*?<span.*?>(.*?)</span>.*?\
<td width=.*?<p class=MsoNormal.*?<span.*?>(.*?)</span>.*?\
<td width=.*?<p class=MsoNormal.*?<span.*?>(.*?)</span>.*?\
<td width=.*?<p class=MsoNormal.*?<span.*?>(.*?)</span>.*?\
<td width=.*?<p class=MsoNormal.*?<span.*?>(.*?)</span>.*?\
<td width=.*?<p class=MsoNormal.*?<span.*?>(.*?)</span>.*?\
<td width=.*?<p class=MsoNormal.*?<span.*?>(.*?)</span>.*?</tr>', re.S)
items = re.findall(pattern, html)
print(type(items))
result = []
for item in items:
result.append((item))
df = pd.DataFrame(data=result[1:],columns = result[0])
print(set(df['拟录取单位']))
ke = ['计算机科学与技术学院', '软件学院']
df = df.drop(index = df[~df['拟录取单位'].isin(ke)].index, axis=0)
print(df)
df = df.sort_values(by = ['拟录取单位','拟录取专业','总分'],ascending=[True,False,False])
print(len(df))
fil = 'D:/2020Ustc拟录取.xlsx'
if os.path.exists(fil):
os.remove(fil)
df.to_excel('D:/2020Ustc拟录取.xlsx')
def writeFile(content):
with open('3.txt', 'a', encoding = 'utf-8') as f:
f.write(json.dumps(content, ensure_ascii = False) + '\n')
f.close()
def main():
# with open('4.txt', 'w', encoding = 'utf-8') as f:
# f.write(html)
# f.close()
# for item in getInf(html):
# print(item)
# writeFile(item)
#s = open("4.txt",encoding='utf-8').read()
#getInf(s)
if __name__ == '__main__':
main()
爬取USTC2021研究生拟录取
发现与去年的格式相比有了一点变化,就对正表表达式做了一点修改;
import requests
import re
import json
import pandas as pd
import sys
import os
# print('当前 Python 解释器路径:')
# print(sys.executable)
def getInf(html):
pattern = re.compile('<tr style=\'mso-yfti-irow.*?\
<td width="9%".*?<p class=MsoNormal.*?<span.*?>(.*?)</span>.*?\
<td width="3%".*?<p class=MsoNormal.*?<span.*?>(.*?)</span>.*?\
<td width="5%".*?<p class=MsoNormal.*?<span.*?>(.*?)</span>.*?\
<td width="21%".*?<p class=MsoNormal.*?<span.*?>(.*?)</span>.*?\
<td width="18%".*?<p class=MsoNormal.*?<span.*?>(.*?)</span>.*?\
<td width="5%".*?<p class=MsoNormal.*?<span.*?>(.*?)</span>.*?\
<td width="6%".*?<p class=MsoNormal.*?<span.*?>(.*?)</span>.*?\
<td width="4%".*?<p class=MsoNormal.*?<span.*?>(.*?)</span>.*?\
<td width="3%".*?<p class=MsoNormal.*?<span.*?>(.*?)</span>.*?\
<td width="4%".*?<p class=MsoNormal.*?<span.*?>(.*?)</span>.*?\
<td width="4%".*?<p class=MsoNormal.*?<span.*?>(.*?)</span>.*?</tr>', re.S)
items = re.findall(pattern, html)
print(type(items))
print(items)
result = []
for item in items:
result.append((item))
print(result)
df = pd.DataFrame(data=result[1:],columns = result[0])
print(set(df['拟录取单位']))
print(df)
df = df.sort_values(by = ['拟录取单位','拟录取专业','总分'],ascending=[True,False,False])
print(len(df))
fil = 'D:/2020Ustc拟录取.xlsx'
if os.path.exists(fil):
os.remove(fil)
df.to_excel('D:/2021Ustc拟录取.xlsx')
def writeFile(content):
with open('3.txt', 'a', encoding = 'utf-8') as f:
f.write(json.dumps(content, ensure_ascii = False) + '\n')
f.close()
def main():
s = open("4.txt",encoding='utf-8').read()
getInf(s)
print(2**3)
if __name__ == '__main__':
url = "http://search.gradschool.ustc.edu.cn/download/yzbgs/2021tkssgs.htm"
kv={'User-Agent':'Mozilla/5.0'}
reponse = requests.get(url, headers=kv)
reponse.encoding = 'gb2312'
html = reponse.text
with open('4.txt', 'w', encoding = 'utf-8') as f:
f.write(html)
f.close()
main()