day6 常见反爬

常见反爬

01 . requests使用代理IP

• 使用代理：给参数proxies赋值 - {‘http’：‘ip端口’}

  import requests
  
  
  def get_html(url):
      headers = {
          'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36'
      }
      # 使用固定的代理ip
      response = requests.get(url, headers=headers,proxies={'https':'119.7.145.225:4558'})
      print(response.text)
      return response.text
  
  if __name__ == '__main__':
      for _ in range(2):
          url = 'https://movie.douban.com/top250'
          get_html(url)
  

04 . selenium使用代理ip

• 使用代理 - --proxy-server=http://IP:端口号 （ip端口是https的ip）

  from selenium.webdriver import Chrome,ChromeOptions
  
  options = ChromeOptions()
  options.add_argument('--proxy-server=http://119.5.36.108:4531')
  b = Chrome(options=options)
  b.get('https://movie.douban.com/top250')
  

03 . requests自动登录

• 在谷歌浏览器中打开网页完成登录操作，然后刷新页面

• 打开当前页面的检查，在network的All选项中找到当前页面的请求，获取请求头request header中的cookie值

• 在用requests发送请求的时候给headers赋值，在headers中添加cookie对应的键值对

  import requests
  
  headers = {
          'cookie':'_zap=8e7efb63-1ac5-4329-b7bc-c1f282b29c90; d_c0="AGDf6WH-lRSPTsCQPiLp82XCJk0tVCHfxqs=|1646443289"; _9755xjdesxxd_=32; YD00517437729195%3AWM_TID=jbZcNMq0oW9FQBAEVRZ%2BvzM%2F7n1J%2BiBA; _xsrf=f1Uz4S6FIenrnXrPFbNYE8yyV750LuTg; Hm_lvt_98beee57fd2ef70ccdd5ca52b9740c49=1646651691,1646652767,1647660780,1647833960; __snaker__id=gUl7lWYKdQ5arwzV; gdxidpyhxdE=tEdyzOzRa0M%2F%2F0RCKvT8XDAHDiSG239517tqczPxd5%2Bd1xn%2FHivZ1YeyrJT%2BXI6fLK6IhdTfd2ysVkupPIXnfI%2FbpJmIHwvJ9a%2BiIJYUSgxytwPgy7jiTL29WHrSb8dD6d53NqOJHSVtH30mu3lSyiHAs7LjEonQeIY%5CwypJJm%5CTIxO8%3A1647834862434; YD00517437729195%3AWM_NI=pAo8OUj0PF501UTE30w49OGQOlb38E%2FHkF4aNhj2%2B9qsR7TlJNFklMOJ67zNN1bk7SJoEH4QUd0FpRRF6phpIdnf2kLR8jpz2Hcy5TY40CwYqQb4TYB5%2Fm8R1PPe6VcFSVQ%3D; YD00517437729195%3AWM_NIKE=9ca17ae2e6ffcda170e2e6eeb8bc67ba9efdb4fc4a978e8eb2d54e868b8b85b53df7a78699d363fc9c978de52af0fea7c3b92ab49fa1d9ce5faaf0a0a7b83cf38b9ca6b23cb5ecf7b0c65b8db4aed4bb4fed98a5a7b76b8eb8a6d3aa33b3ad9ed8d7688a928a8cef478d96fad8f9468eefae8cb6598ab18ba9b75b97bc00a7f35a929afa91eb7c8bbca287b161908ef88bf372f298ad9ab8609bec00a4f653b38faed5d75382ebe18ebb5af390a086b55ea19a9db7d437e2a3; SESSIONID=yOdEeqTwAta8V8XWBjvnmtKDToSOOUp1iuQLjMBa4GF; JOID=VFwcCkxLLCzWctXrXETP8w0UJH1HMG5InRamsisLWkDuKuyxBvLSFbtx2uZSO0OnvaGm3XBgbSu3Yf4Q4WaHeEs=; osd=VVAdCkJKIC3WfNTnXUTB8gEVJHNGPG9IkxeqsysFW0zvKuKwCvPSG7p92-ZcOk-mva-n0XFgYyq7YP4e4GqGeEU=; captcha_session_v2=2|1:0|10:1647834064|18:captcha_session_v2|88:R3RRbjdOL1ozK0ZleVJnVXpMS2dabVIrT21KZVFJL3BQMm5NVUF0cUxsdm96dGJDc0tUejlEK09MR053azBQcA==|9e05def29c63ea0f47e3e4b7d049950179b8b19120e731b53cc52e9171e29cad; captcha_ticket_v2=2|1:0|10:1647834417|17:captcha_ticket_v2|704:eyJ2YWxpZGF0ZSI6IkNOMzFfOEVoZHBrRWJrLmFFWlpjUVJuX2dQcUsxek1hdjJuRmowNHguREp1ZENWWnNCNjhvejBxbjBSOEd0bUNBSGpNb3pwZkVhOV9vWUVkOC4xRm0ubi5LTVlqbndQdzlveU5PZFp3VnA4bnI2SHF2QzBLb0ZoUWdkeDAyVTFJRndUVjk0VHRJSzRtazRGSUdwOC1kRWp5dEJVUkZsUmg0WDFmTHMyUURsMmo4NWFUMlN2VUljS2FTWE5zWTcyVVJnMlA4OEs3SnBuTmptUklZSV9jU1VHLmoxRHBFYWRlN0lpNS1aTzdObENvZnZRX3R4LVI0WnNZOFlYaExFTzVkN19hOVV5RHItYXdwOW9VWkRqQzBIaXZ2VXJrYnRsajdkZUJpLjZ6ajEwbVd4VnplWE1ScGdSa3QyUk1uSkdSbXd0TC5iTlRfVEVJd2ZZcW1Fd29fbXhvb0VzcHV3NTE0cC5IdjVsVkd3Zlp3d2NYdjhhSHc5SXpKSW9VWVZjRS5kQ0ZIQnpaSEdPWFFjTlF2cHVDX0FKWHBoR1AwaHI4X05YVFhXYVlSWWFjazdrTU9NeU9XQm5BT2x4Ui4xYk44eFlTV3paNE5MUE5VOTI0NGpZcUVVVm5reTBFcERiZDFrT0toNWkwTldZWFhhSHdGZFNXcktkUFB0VHlrSU5hMyJ9|d68fa76d4f7f7246f17cceb14c65c623020b69017b839f58ca453ffc79ccc20a; z_c0=2|1:0|10:1647834467|4:z_c0|92:Mi4xTHRLLU53QUFBQUFBWU5fcFlmNlZGQ1lBQUFCZ0FsVk5ZME1sWXdDTTAyd0g3SlZYWUdWb0xQS1cwLVBOQkJtTTR3|60716c2b69507315628a2c721ca0d323dfa3be95b5f75a815ee4f5cf6add389b; q_c1=617cb32a434147c7bcda4cd05dbadaa8|1647834467000|1647834467000; Hm_lpvt_98beee57fd2ef70ccdd5ca52b9740c49=1647834471; tst=r; NOT_UNREGISTER_WAITING=1; KLBRSID=e42bab774ac0012482937540873c03cf|1647834513|1647833956',
          'user-agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36'
      }
  
  response = requests.get('https://www.zhihu.com/',headers=headers)
  
  print(response.text)
  

04 . selenium自动登录

• 穿件浏览器对象，打开需要自动登录的网站

• 手动完成登录操作

• 获取cookie数据并保存到本地文件中

  from selenium.webdriver import Chrome,ChromeOptions
  
  b = Chrome()
  b.get('https://www.taobao.com/')
  input('是否完成:')
  cookies = b.get_cookies()
  open('files/taobao.txt', 'w',encoding='utf-8').write(str(cookies))
  

05 . selenium自动登录使用cookie

• 创建浏览器对象打开网页

• 添加本地保存的cookie信息

• 重新打开网页

• 进行后续其他操作

  from selenium.webdriver import Chrome,ChromeOptions
  from selenium.webdriver.common.keys import Keys
  
  b = Chrome()
  b.get('https://www.taobao.com/')
  cookie_list = eval(open('files/taobao.txt',encoding='utf-8').read())
  for cookie in cookie_list:
      b.add_cookie((cookie))
  
  b.get('https://www.taobao.com/')
  
  search = b.find_element_by_id('q')
  search.send_keys('鞋子')
  search.send_keys(Keys.ENTER)
  

06 . 字体反爬

import requests
from bs4 import BeautifulSoup
import re

headers = {
    'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36'
}
url = 'https://mapi.guazi.com/car-source/carList/pcList?page=1&pageSize=12&city_filter=12&city=12&guazi_city=12&tag_types=18&versionId=0.0.0.0&osv=IOS&platfromSource=wap'
response = requests.get(url, headers=headers)

table = {
    '0xe1d0': '7', '0xe325': '4', '0xe41d': '1', '0xe52e': '9', '0xe630': '2', '0xe76e': '8',
    '0xe891': '5', '0xe9ce': '0', '0xeaf2': '3', '0xec4c': '6', '0xf88a': '7'
}
# hex(十进制数)  -  将10进制转换成16进制
all_car = response.json()['data']['postList']
for car in all_car:
    price = car['price'].split(';')
    new_price = ''
    for x in price:
        if x.startswith('&#'):
            new_price += table[hex(int(x[2:]))]
        elif x.startswith('.&#'):
            new_price += '.' + table[hex(int(x[3:]))]
        else:
            new_price += x
    print(new_price)