py爬虫2

中国大学排名

输入大学排名url
输出大学排名信息的屏幕输出
测试是否能爬：robots.text

1、获取网页内容
2、利用信息到合适的数据结构
3、利用数据结果展示并输出

程序设置

import requests
from bs4 import BeautifulSoup
def getHTMLText(url): /爬取url信息，其中页面返回其他程序
return “”

def fillUnivList(ulist, html): /提取html关键数据添加到列表
pass

def printUnivList(ulist, num):/内容
print(“Suc” + str(num))
def main(): /需要的替换标签
uinfo = []
url = ‘http://www.zuihaodaxue.cn/Greater_China_Ranking2019_0.html’
html = getHTMLText(url) /使用地址
fillUnivList(uinfo, html) /功能
printUnivList(uinfo, 20) # 20 univs/打印大学信息
main() /主体

getHTMLText
try:
r = requests.get(url, timeout = 30)
r.raise_for_status() /产生异常
r.encoding = r.apparent_encoding / 修改编码
return r.text /返回
except:

fillUnivList /提取关键数据
def fillUnivList(ulist, html): 解析tbody
soup = BeautifulSoup(html, “html.parser”)
for tr in soup.find(‘tbody’).children: 查找tr标签的类型中的td
if isinstance(tr, bs4.element.Tag):为了运行引用bs4
tds = tr(‘td’) 定义列表
ulist.append([tds[0].string, tds.string, tds[2].string]) 列表

找列表中的相关功能

def printUnivList(ulist, num):
print("{:^10}\t{:6}\t{:^10}",format(“排名”, “学校名称”, “总分”))
for i in range(num): /{}：计算机{}的cpu占用=。rormat( , , )可以用012占用
u = ulist[i]
print("{:^10}\t{:6}\t{:^10}".format(u[0], u[1], u[2])) /替换

import requests
from bs4 import BeautifulSoup
import bs4

def getHtmlText(url):
try:
r = requests.get(url, timeout=30)
r.raise_for_status()
r.encoding = r.apparent_encoding
return r.text
except:
return “”

def fillUnivList(ulist, html):
soup = BeautifulSoup(html, “html.parser”)
for tr in soup.find(‘tbody’).children:
if isinstance(tr, bs4.element.Tag):
tds = tr(‘td’)
ulist.append([tds[0].string, tds[1].string, tds[2].string])

def printUnivList(ulist, num):
print("{:^10}\t{:6}\t{:^10}".format(“排名”, “学校名称”, “总分”))
for i in range(num):
u=ulist[i]
print("{:^10}\t{:6}\t{:^10}".format(u[0], u[1], u[2]))
def main():
uinfo = []
url = ‘http://www.zuihaodaxue.com/Greater_China_Ranking2019_0.html’
html = getHtmlText(url)
fillUnivList(uinfo, html)
printUnivList(uinfo, 20)
main()

中文字符占位
chr（12288）
添加输出模板的变量
tplt = "{“^10}\t{:6}\t{:^10}”
import requests
from bs4 import BeautifulSoup
import bs4

def getHtmlText(url):
try:
r = requests.get(url, timeout=30)
r.raise_for_status()
r.encoding = r.apparent_encoding
return r.text
except:
return “”

def fillUnivList(ulist, html):
soup = BeautifulSoup(html, “html.parser”)
for tr in soup.find(‘tbody’).children:
if isinstance(tr, bs4.element.Tag):
tds = tr(‘td’)
ulist.append([tds[0].string, tds[1].string, tds[3].string])

def printUnivList(ulist, num):
tplt = “{0:^10}\t{1:{3}10}\t{2:^10}”
print(tplt.format(“排名”, “学校名称”, “总分”, chr(12288)))
for i in range(num):
u=ulist[i]
print(tplt.format(u[0], u[1], u[2], chr(12288)))
def main():
uinfo = []
url = ‘http://www.zuihaodaxue.com/Greater_China_Ranking2019_0.html’
html = getHtmlText(url)
fillUnivList(uinfo, html)
printUnivList(uinfo, 20)
main()

正则表达式

字符表示

.表示任何单个字符
[ ]字符范围
[^ ]非字符集表示非^字符
扩展 前一个字符0次或无限次表示abc=ab、abc、abcc
+扩展 前一个字符1次或无限次表示abc+=abc、abcc、abccc
？扩展 前一个字符0次或1次 表示abc？=ab、abc
！左右表达式任意一个 表示abc|def =abc、def
{n}扩展前一个字符m次 表示ab{2}=abb
{m,n}扩展前一个字符m至n次 ab{1,2}c=abc、abbc
^ 匹配字符串开头 ^abc表示abc且在一个字符串的开头
$匹配字符串结尾abc$表示abc且在一个字符串的结尾
()分组标记，内部只能使用|操作符（abc）=abc，（abc|def）=abc、def |表示或
\d 数字，等价于[0-9]
\w 单词字符，等价于[A-Za-z0-9_]

实例

正则表达式 对应字符串
P(Y|YT|YTHO)?N ’PN‘ ‘PYN’ ‘PYTN’ ‘PYTHN’ ‘PYTHON’
PYTHON+ ‘PYTHON’ ,‘PYTHONN’,‘PYTHONNN’…
PY[TH]ON ‘PYTON’ ‘PYHON’
PY[^TH]?ON ‘PYON’ ‘PYaON’ ‘PYbON’ ‘PYcON’…
PY{:3}N ‘PN’ ‘PYN’ ‘PYYN’ ‘PYYYN’
¹+$ 由26个字母组成的字符串
² 由26个字母和数字组成的字符串
^-?\d+$ 整数形式的字符串
³[1-9][0-9]$ 正整数形式的字符串
[1-9]\d{5} 中国境内邮政编码6位
[\u4e00- \u9fa5] 匹配中文字符
\d{3}- \d{8}|\d{4}-\d{7} 国内电话号码，010-68913536
\d+.\d+.\d+.\d+ ip字符串表达形式
\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} ip地址分四段，每段0-255
[1-9]?\d 0-99
1\d{2} 100-199
2[0-4]\d 200-249
25[0-5] 250-255

使用import re

re.search返回match结果
当正则表达式出现<转义符>时使用raw string
用print输出

import re
match = re.search(r’[1-9]\d{5}’, ‘BIT 100081’)

if match:
print(match.group(0))

100081

re.match
功能从一个字符串的开始位置起匹配正则表达式，返回match对象（原生字符串，待匹配，使用控制标记）要倒着写参数

import re
match = re.match(r’[1-9]\d{5}’, ‘100081 BIT’)

if match:
match.group(0)

‘100081’

re.findall
功能以列表类型返回

import re

Is = re.findall(r’[1-9]\d{5}’, ‘BIT100081 TSU100084’)

Is

[‘100081’, ‘100084’]

re.split
将字符串按照正则表达式匹配结果进行分割，返回列表类型

import re
re.split(r’[1-9]\d{5}’, ‘BIT100081 TSU100084’)

[‘BIT’, ’ TSU’, ‘’]

re.split(r’[1-9]\d{5}’, ‘BIT100081 TSU100084’,maxsplit=1)

[‘BIT’, ’ TSU100084’] /maxsplit=1表示分割出1个

re.finditer
获得迭代表达式结果
for m in re.finditer(r’[1-9]\d{5}’, ‘BIT100081 TSU100084’):
if m :
print(m.group(0))

100081
100084

re.sub
表示在一个字符串中所替换的所有匹配正则表达式的子串，返回替换后的字符串

import re

re.sub(r’[1-9]\d{5}’, ‘zipcode’, ‘BIT100081 TSU100084’)

‘BITzipcode TSUzipcode’
/替换了

操作等价用法
rst = re.search(r’[1-9]\d{5}’, ‘BIT 100081’)
面型对象用法
pat = re.comple(r’[1-9]\d{5}’)
rst = pat.search(‘BIT 100081’)
对象是regex 调用
regex = recompile（r‘[1-9]\d{5}’）
对应正则表达式方式
regex.search()
regex.match()
regex.finditer
regex.sub
match

match = re.search(r’[1-9]\d{5}’, ‘BIT100081’)

if match:
print(match.group(0))

100081

type(match)

<class ‘re.Match’>

.strung 待匹配文本
.re 匹配时受用的pattern对象（正则表达式）
.pos正则表达式搜索文本的开始位置
.endpos正则表达式搜索文本的结束位置
.group(0)获得匹配的字符串
.start（）匹配字符串在原始字符串开始的位置
.end()匹配字符串在原始字符串的结束位置
.span（）返回（.star .end）

m = re.search(r’[1-9]\d{5}’, ‘BIT100081 TAU100084’)
m.string
‘BIT100081 TAU100084’

m.re
re.compile(’[1-9]\d{5}’)

m.pos
0

m.endpos /从0 到19开始搜索
19

m.group(0)/匹配字符串
‘100081’

m.start
<built-in method start of re.Match object at 0x0431C8E0>

m.start()
3

m.end()
9

贪婪匹配默认最大 最小匹配加？
最小
*?前一个字符0次或无限次扩展，最小匹配
+？前一个字符1次或无限次扩展，最小匹配
？？前一个字符0次或1次扩展最小匹配
{m,n}扩展前一个字符m至n含n，最小匹配

京东实例

京东

import requests
r = requests.get(“https://item.jd.com/100002795959.html”)
r.status_code
200

r.encoding
‘gbk’

r.text[:1000]
‘\n\n\n \n \n \n \n \n \n \n \n \n \n \n <link rel="dns-prefetch’

亚马逊

import requests
r = requests.get(“https://www.amazon.cn/dp/B07YTH6VMD/ref=s9_acsd_ri_bw_r2_r0_0_i?pf_rd_m=A1U5RCOVU0NYF2&pf_rd_s=merchandised-search-3&pf_rd_r=K51BNXBQSJQ62QAP3FKV&pf_rd_t=101&pf_rd_p=969dd71e-181f-4bfb-bcca-183a2a6a69de&pf_rd_i=143231071”)
r.status_code
503

r.encoding
‘ISO-8859-1’

r.encoding = r.apparent_encoding
r.text
r.request.headers
{‘User-Agent’: ‘python-requests/2.22.0’, ‘Accept-Encoding’: ‘gzip, deflate’, ‘Accept’: ‘/’, ‘Connection’: ‘keep-alive’}

r = requests.get(url,headers = kv)
r.status_code
200

r.request.headers
{‘user-agent’: ‘Mozilla/5.0’, ‘Accept-Encoding’: ‘gzip, deflate’, ‘Accept’: ‘/’, ‘Connection’: ‘keep-alive’}

r.text[:1000]
‘\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n\n\n\n\n\n\n\n\n \n \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n \n\n\n \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n\n\n\n\n <!doctype html>\n \n\n

百度关键词接口
http：//www.baidu.com/s?wd=keyword
360关键词接口
http：//www.so.com/s?q=keyword

百度代码

import requests
kv = {‘wd’:‘python’}
r = requests.get(“http://baidu.com/s”,params = kv)
r.status_code
200

r.request.url
‘https://wappass.baidu.com/static/captcha/tuxing.html?&ak=c27bbc89afca0463650ac9bde68ebe06&backurl=https%3A%2F%2Fwww.baidu.com%2Fs%3Fwd%3Dpython&logid=7698784756032329280&signature=f7fefd9e394e3d963e073631d11d44ee&timestamp=1581238835’

len(r.text)
1519

图片爬取

import requests
path = “C:/Users/zsnzd”
url = “https://image.baidu.com/search/detail?ct=503316480&z=0&ipn=d&word=%E5%9B%BE%E7%89%87&hs=2&pn=1&spn=0&di=180730&pi=0&rn=1&tn=baiduimagedetail&is=0%2C0&ie=utf-8&oe=utf-8&cl=2&lm=-1&cs=3173584241%2C3533290860&os=1133571898%2C402444249&simid=3493630544%2C246115770&adpicid=0&lpn=0&ln=30&fr=ala&fm=&sme=&cg=&bdtype=0&oriquery=%E5%9B%BE%E7%89%87&objurl=http%3A%2F%2Fa0.att.hudong.com%2F78%2F52%2F01200000123847134434529793168.jpg&fromurl=ippr_z2C%24qAzdH3FAzdH3Fp7rtwg_z%26e3Bkwthj_z%26e3Bv54AzdH3Ftrw1AzdH3Fwa_0b_cd_a8daaaaa8dnb908n99n9cdl0ln8mb_3r2_z%26e3Bip4s&gsm=2&islist=&querylist=”
r = requests.get(url)
r.status_code
200

把图片保存文件

with open(path,‘wb’)f:

SyntaxError: invalid syntax

with open(path,‘wb’) as f:
f.write(r.content)

123649

f.close()

获取html

import requests
r = requests.get(“https://python123.io/ws/demo.html”)
r.text
‘\r\n\r\n

The demo python introduces several python courses.

\r\n

Python is a wonderful general-purpose programming language. You can learn Python from novice to professional by tracking the following courses:\r\nBasic Python and Advanced Python.

\r\n’

demo = r.text
from bs4 import BeautifulSoup
soup = BeautifulSoup(demo , “html.parser”)
print(soup.prettify())

The demo python introduces several python courses.

Python is a wonderful general-purpose programming language. You can learn Python from novice to professional by tracking the following courses: Basic Python and Advanced Python .

>>>

---

1. A-Za-z ↩︎

2. A-Za-z0-9 ↩︎

3. 0-9 ↩︎