爬虫初识

爬虫

博客参考

1. 开源模块：

• requests
  response = requests.get(“url”)
  response.text
  response.content
  response.encoding = response.apparent_encoding # 使用网站编码
  response.status_code
  response.cookies.get_dict()
  requests.get(“url”, cookie={‘xx’: ‘yy’})
  requests.request方法参数：
  • method: 提交方式
  • url: 提交URL
  • params: 在URL中传递的参数

requests.request(
	method='GET',
	url='http://www.baidu.com',
	params={'k1':'v1','k2':'v2'}
)

-data: 在请求体重传递的参数

requests.request(
	method='GET',
	url='http://www.baidu.com',
	data={'k1':'v1','k2':'v2'}
)

-json: 在请求体重传递的参数(字典中嵌套字典时使用)

requests.request(
	method='GET',
	url='http://www.baidu.com',
	data={'k1':'v1','k2':{'k3':'v3'}}
)

-headers: 请求头

requests.request(
	method='POST',
	url='https://github.com/login',
	data={'k1':'v1','k2':{'k3':'v3'}},
	headers={
		'Referer': 'https://github.com/',
		'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36'
	}
)

-cookies: Cookies
-session: 用于保存客户端历史访问信息

import requests
def requests_session():
    session = requests.Session()

    ### 1、首先登陆任何页面，获取cookie
    i1 = session.get(url="http://dig.chouti.com/help/service")

    ### 2、用户登陆，携带上一次的cookie，后台对cookie中的 gpsd 进行授权
    i2 = session.post(
        url="http://dig.chouti.com/login",
        data={
            'phone': "8615131255089",
            'password': "xxxxxx",
            'oneMonth': ""
        }
    )

    i3 = session.post(
        url="http://dig.chouti.com/link/vote?linksId=8589623",
    )
    print(i3.text)

• beautisoup
  安装：pip install requests beautifulsoup4

饭粒1：爬取汽车之家新闻页

import requests
from bs4 import BeautifulSoup

url = 'https://www.autohome.com.cn/all/'
response = requests.get(url)
response.encoding = response.apparent_encoding   # 使用网站编码
soup = BeautifulSoup(response.text, features='html.parser')
content = soup.find(id='auto-channel-lazyload-article')
# 组合条件：soup.find('div', class='c1')
li_list = content.find_all('li')   # 返回列表
for i in li_list:
    a = i.find('a')
    if a:
        # 新闻URL
        url = a.attrs.get('href').strip('//')
        print(url)
        # 新闻标题
        text = a.find('h3').text   # 获取标签内容
        print(text)
        # 新闻图片URL
        img_url = 'http://' + a.find('img').attrs.get('src').strip('//')
        # 保存图片
        img_response = requests.get(url=img_url)
        file_name = 'image' + os.path.sep + str(uuid.uuid4()) + '.jpg'
        with open(file_name, 'wb') as f:
            f.write(img_response.content)

饭粒2：登陆github

import requests
from bs4 import BeautifulSoup

# 获取token
r1 = requests.get('https://github.com/login')
s1 = BeautifulSoup(r1.text, 'html.parser')
token = s1.find(name='input', attrs={'name': 'authenticity_token'}).get('value')
r1_cookie_dict = r1.cookies.get_dict()

# 将用户名密码token发送到服务端，post
"""
github登陆默认post提交的信息：
utf8:✓
authenticity_token:ollV+avLm6Fh3ZevegPO7gOH7xUzEBL0NWdA1aOQ1IO3YQspjOHbfnaXJOtVLQ95BtW9GZlaCIYd5M6v7FGUKg==
login:asdf
password:asdf
commit:Sign in
"""
r2 = requests.post(
    'https://github.com/session',
    data={
        "utf8": '✓',
        "authenticity_token": token,
        'login': 'afengbo',
        'password': 'woshimima',
        'commit': 'Sign in'
    },
    cookies=r1_cookie_dict
)
r2_cookie_dict = r2.cookies.get_dict()
cookie_dict = {}
cookie_dict.update(r1_cookie_dict)
cookie_dict.update(r2_cookie_dict)

# 获取个人信息页面
r3 = requests.get(
    url='https://github.com/settings/emails',
    cookies=cookie_dict
)
print(r3.text)