Python 爬虫 实例

一.爬取汽车之家上咨询文章的标题及链接

import requests,bs4

response=requests.get("https://www.autohome.com.cn/all/#pvareaid=3311230")
response.encoding=response.apparent_encoding
soup=bs4.BeautifulSoup(response.text,features="html.parser")
target=soup.find(id="auto-channel-lazyload-article")
li_list=target.find_all("li")
for obj in li_list:
    i=obj.find("a")
    if i:
        print(i.find("h3").text)
        img_url=i.attrs.get("href")
        print(img_url)

二.自动登录抽屉新热榜并点赞

import requests

r1=requests.get("https://dig.chouti.com/")
r1_cookie_dict=r1.cookies.get_dict()#获取dict形式的Cookies

post_dict={
    "phone":12345678901,
    "password":"woshinibaba",
    "OneMonth":1#将登录状态保留1个月
}
r2=requests.post(#进行登录操作
    "https://dig.chouti.com/login",
    data=post_dict,
    cookies=r1_cookie_dict#发送过去进行授权
)

r3=requests.post(
    "https://dig.chouti.com/link/vote?linksId=11832246",
    cookies=r1.cookies.get("gpsd")#用于证明自己已经登陆
    #注:登录成功后,Server会给r1中的gpsd授权,而r2中的gpsd没有任何用处(这是1个例外,通常使用r2中的Cookies)
)

注:目前已改版,该实例无法直接使用

三.自动登录GitHub并获取自己仓库中所有项目的名称

import requests,bs4

#访问登陆页面,获取authenticity_token:
i1=requests.get('https://github.com/login')
soup1=bs4.BeautifulSoup(i1.text,features='lxml')
tag=soup1.find(
  name='input',
  attrs={'name':'authenticity_token'}
)
authenticity_token=tag.get('value')
#获取第1次访问的Cookie:
c1=i1.cookies.get_dict()
i1.close()

#携带authenticity_token/username/pwd码等信息,发送用户验证:
form_data={
  "authenticity_token":authenticity_token,
  "utf8":"",
  "commit":"Sign in",
  "login":"12345678@gmail.com",
  "password":"88888888"
}
i2=requests.post(
  'https://github.com/session',
  data=form_data,
  cookies=c1
)
#获取第2次访问的Cookie:
c2=i2.cookies.get_dict()
#把c2中的内容更新到c1中(更新后c1包含2次Cookie中的所有内容):
c1.update(c2)

#访问自己的项目仓库:
i3=requests.get(
  'https://github.com/settings/repositories',
  cookies=c1
)
soup3=bs4.BeautifulSoup(i3.text,features='lxml')
list_group=soup3.find_all(name='div',class_='Box-row simple public fork js-collab-repo')
for child in list_group:
    name=child.find(name="a").attrs.get("href")
    print(name)#打印自己所有项目的名称

四.自动登录知乎

• 登录次数过多后,会要求输入图片中的验证码(可以进行图像识别,可以使用插件或购买API,也可以下载图片并手动输入)

import time,requests,bs4

#访问登录页面,获取xsrf属性的值(用于授权登录):
session=requests.Session()
i1=session.get(
    url='https://www.zhihu.com/#signin',
    headers={
        'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36',
    }
)
soup1=bs4.BeautifulSoup(i1.text,'lxml')
xsrf_tag=soup1.find(name='input',attrs={'name':'_xsrf'})
xsrf=xsrf_tag.get('value')

#下载包含验证码的图片:
current_time=time.time()
i2=session.get(
    url='https://www.zhihu.com/captcha.gif',
    params={
        'r':current_time,
        'type':'login'
    },
    headers={
        'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36',
    }
)
with open('zhihu.gif','wb') as f:
    f.write(i2.content)

#输入验证码,进行登录:
captcha=input('请打开zhihu.gif文件,查看并输入验证码:')
form_data={
    "_xsrf":xsrf,
    'password':'xxooxxoo',
    "captcha":'captcha',
    'email':'424662508@qq.com'
}
i3=session.post(
    url='https://www.zhihu.com/login/email',
    data=form_data,
    headers={
        'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36',
    }
)

五.自动登录博客园

• 登录时会将用户名和密码加密后再发到后台

import re,json,base64,rsa,requests

def js_encrypt(text):#进行加密(cnblogs使用插件JSEncrypt进行加密)
    #加密使用的公钥:
    b64der='MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCp0wHYbg/NOPO3nzMD3dndwS0MccuMeXCHgVlGOoYyFwLdS24Im2e7YyhB0wrUsyYf0/nhzCzBK8ZC9eCWqd0aHbdgOQT6CuFQBMjbyGYvlVYU2ZP7kG9Ft6YV6oc9ambuO7nPZh+bvXH0zDKfi02prknrScAKC0XhadTHT3Al0QIDAQAB'
    der=base64.standard_b64decode(b64der)
    pk=rsa.PublicKey.load_pkcs1_openssl_der(der)
    v1=rsa.encrypt(bytes(text, 'utf8'), pk)
    value=base64.encodebytes(v1).replace(b'\n', b'')
    value=value.decode('utf8')
    return value

session=requests.Session()
i1=session.get('https://passport.cnblogs.com/user/signin')
rep=re.compile("'VerificationToken':'(.*)'")
v=re.search(rep,i1.text)
verification_token=v.group(1)

form_data={
    'input1':js_encrypt('wptawy'),
    'input2':js_encrypt('asdfasdf'),
    'remember':False
}
i2=session.post(url='https://passport.cnblogs.com/user/signin',
    data=json.dumps(form_data),
    headers={
        'Content-Type':'application/json; charset=UTF-8',
        'X-Requested-With':'XMLHttpRequest',
        'VerificationToken':verification_token
    }
)

i3=session.get(url='https://i.cnblogs.com/EditDiary.aspx')
print(i3.text)