模拟提交表单、抓取动态数据、识别验证码

模拟提交表单、抓取动态数据、识别验证码

模拟提交表单

import requests
from bs4 import BeautifulSoup


def main():
    resp = requests.get('https://github.com/login')
    if resp.status_code != 200:
        return
    # 得到cookies
    cookies = resp.cookies.get_dict()
    print(cookies)
    soup = BeautifulSoup(resp.text, 'lxml')
    # token
    # 防跨站身份伪造
    urf8_value = \
        soup.select_one('form input[name=utf8]').attrs['value']
    authenticity_token = \
        soup.select_one('form input[name=authenticity_token]').attrs['value']
    print(urf8_value)
    print(authenticity_token)
    data = {
        'utf8': urf8_value,
        'authenticity_token': authenticity_token,
        'login': '',
        'password': ''
    }


    # 模拟登陆提交表单
    resps = requests.post('https://github.com/session', data=data, cookies=cookies)
    print(resps.text)
    # 如果要上传文件
    # files = {
    #     'file1': open(),
    #     'file2': open()
    # }
    # resps = requests.post('https://github.com/session', data=data, cookies=cookies, files=files)


if __name__ == '__main__':
    main()

使用robobrowser

# pip install robobrowser 可以模拟浏览器提交表单
import robobrowser
import requests


def main():

    # robobrowser 内部是用bs4实现的
    b = robobrowser.RoboBrowser(history=True, parser='lxml')
    b.open(('https://github.com/login'))
    # 选择页面要提交的表单
    f = b.get_form(action='/session')
    f['login'].value = ''
    f['password'].value = ''
    resps = b.submit_form(f)
    print(resps)
    for a_tag in b.select('a[href]'):
        print(a_tag.attrs['href'])



if __name__ == '__main__':
    main()

抓取动态数据

# pip install selenium 一个测试用的工具，可以模拟浏览器操作
from selenium import webdriver
from bs4 import BeautifulSoup
from selenium.webdriver.common.keys import Keys


def main():
    # 可以驱动浏览器
    # 要先下浏览器驱动
    # 第一种方法
    # driver = webdriver.Chrome(executable_path='驱动的路径')
    # 第二种方法
    # 放在path环境变量中
    # 第三种方法
    # 放在已放过path环境变量的文件中
    driver = webdriver.Chrome()
    driver.get('https://v.taobao.com/v/content/live?catetype=704&from=taonvlang')
    # 搜索到要输入内容的标签
    elem = driver.find_element_by_css_selector('input[placeholder=输入关键词搜索]')
    # 向标签内传入数据
    elem.send_keys('运动')
    # 向标签发送回车命令
    elem.send_keys(Keys.ENTER)
    soup = BeautifulSoup(driver.page_source, 'lxml')
    for img_tag in soup.body.select('img[src]'):
        print(img_tag.attrs['src'])


if __name__ == '__main__':
    main()

识别验证码

# pip install pillow 对图片进行处理
# https://pillow.readthedocs.io/en/latest/handbook/index.html
from io import BytesIO

from PIL import Image, ImageFilter
# pip install pytesseract 光学文字识别 同时要下一个tesseract软件
from pytesseract import image_to_string
import requests
import base64


def main():
    # 处理过得图片才可以更好的识别其中内容，所以要用pil来处理图片
    img = Image.open(open('鸣人仙人模式1.jpg', 'rb'))
    # 高斯滤镜
    # img2 = img.filter(ImageFilter.GaussianBlur)
    # img2.save(open('hhe.jpg', 'wb'))

    img1 = Image.open(open('鸣人仙人模式1.jpg', 'rb'))
    img3 = img1.point(lambda x:0 if x < 128 else 255)
    img3.save(open('heheda.jpg', 'wb'))
    # 训练网址
    # https://pp19dd.com/tesseract-ocr-chopper/?i=ocr6DLNCa
    # https://github.com/REMitchell/tesseract-trainer/blob/master/trainer.py
    # 或者用云打码、阿里云市场、聚合数据等第三方api 工作中推荐这种用法
    # 进行光学识别转换成字符串
    # print(image_to_string(img3))
    # 转化成base64格式调第三方接口
    base64.b64encode(BytesIO(img3))

    resp = requests.get('https://pin2.aliyun.com/get_img?type=150_40&identity=mailsso.mxhichina.
                        com&sessionid=k0xHyBxU3K3dGXb59mP9cdeTXxL9gLHSTKhRZCryHxpOoyk4lAVuJhgw==')
    img4 = Image.open(BytesIO(resp.content))
    img4.save('hello.jpg')
    print(image_to_string(img4))
    print(base64.b64encode(resp.content))


if __name__ == '__main__':
    main()