1)网络想要顺畅大量获取数据,逆向工程是不可少的,逆向要消耗太多的精力,怎样守护好自己的知识呢,下面我给出我的一段代码,来启迪大家的心智。
def get_anti_content(): try: res = requests.get("http://你的网址/project/get_pdd/mms/anti-content/") data = res.json()["encrypt_data"] path = os.path.join(os.getcwd(), 'cate', 'aes_buddha.exe') anti = " ".join([path, '-data', data]) anti = os.popen(anti).read() return anti except Exception as e: return ''
可以把一部分核心代码放在云端,云端请求的数据会是一个加密的json。本地用golang来解密请求的json。记住机密的秘钥直接在go程序里,go程序可以设置好过期时间,也可以设置成只能在某台电脑用。 喜欢此代码,可以私信我,大家共同成长。
2)有的网站第一次采集就会出现访问的滑动条验证,采集再频繁就在图片上出现干扰的汉子,让大家进行相应汉子的点选验证。某音会把验证通过或者不通过的信息写在cookie中的s_v_web_id中,会把这个值当成fp,要请求数据必须有正确并且通过的滑条fp才能请求数据。
处理汉子点选验证还是有一定难度,怎样尽可能不出现耟选验证呢?经验告诉我们,当我们找一台新电脑,第一次访问别人网站时,一般最多出现滑条验证,访问多后才会出现点选汉子验证。所以在请求初期就清空cookie目录,让本地电脑看上去是新电脑,就会解决这个问题。
下面是参考代码:
# coding:utf-8 import datetime import math import random import shutil import sys import os # 获取当前文件的目录 from urllib.parse import urlparse import cv2 import numpy as np import requests from selenium.webdriver import ActionChains import api import cooke_path_1 import re import time from selenium import webdriver from selenium.webdriver.common.by import By from fetch_upload_1 import UpData import json # 获取当前文件的目录 from req_login import AES_Decrypt cur_path = os.path.abspath(os.path.dirname(__file__)) sys.path.append(cur_path) cookie_path = cooke_path_1.get_cooke_path_1() get_chromedriver_path = cooke_path_1.get_chromedriver_path() try: shutil.rmtree(cookie_path) print("directory removed successfully") except OSError as e: print("Error: %s : %s" % (cookie_path, e.strerror)) if not os.path.exists(cookie_path): os.makedirs(cookie_path)
3)大招来了,处理滑动条的核心代码,大家参考一下
class Douyin_Slider(object):
def __init__(self, bg, gap):
bg_size = (340, 212)
gap_size = (68, 68)
self.img_dir = os.path.join(os.getcwd(), 'image')
self.bg = self.get_img_path(bg, 'bg', bg_size)
self.gap = self.get_img_path(gap, 'gap', gap_size)
self.out = os.path.join(self.img_dir, 'out.jpg')
def get_img_path(self, img_path, img_name, resize):
headers = {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;"
"q=0.8,application/signed-exchange;v=b3;q=0.9",
"Accept-Encoding": "gzip, deflate, br",
"Accept-Language": "zh-CN,zh;q=0.9,en-GB;q=0.8,en;q=0.7,ja;q=0.6",
"Cache-Control": "max-age=0",
"Connection": "keep-alive",
"Host": urlparse(img_path).hostname,
"Referer": "https:/网址/",
# "Upgrade-Insecure-Requests": "1",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0",
}
resp = requests.get(url=img_path, headers=headers)
if resp.status_code == 200:
img_path = f'./image/{img_name}.jpg'
image = np.asarray(bytearray(resp.content), dtype="uint8")
image = cv2.imdecode(image, cv2.IMREAD_COLOR)
if resize:
image = cv2.resize(image, dsize=resize)
# pass
cv2.imwrite(img_path, image)
return img_path
else:
print('下载失败,状态码为:{}'.format(resp.status_code))
@staticmethod
def clear_white(img):
img = cv2.imread(img)
rows, cols, channel = img.shape
min_x = 255
min_y = 255
max_x = 0
max_y = 0
for x in range(1, rows):
for y in range(1, cols):
t = set(img[x, y])
if len(t) >= 2:
if x <= min_x:
min_x = x
elif x >= max_x:
max_x = x
if y <= min_y:
min_y = y
elif y >= max_y:
max_y = y
img1 = img[min_x:max_x, min_y: max_y]
return img1
@staticmethod
def image_edge_detection(img):
edges = cv2.Canny(img, 100, 200)
return edges
def template_match(self, tpl, target):
th, tw = tpl.shape[:2]
result = cv2.matchTemplate(target, tpl, cv2.TM_CCOEFF_NORMED)
# 寻找矩阵(一维数组当作向量,用Mat定义) 中最小值和最大值的位置
min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(result)
tl = max_loc
br = (tl[0] + tw, tl[1] + th)
# 绘制矩形边框,将匹配区域标注出来
# target:目标图像
# tl:矩形定点
# br:矩形的宽高
# (0,0,255):矩形边框颜色
# 1:矩形边框大小
cv2.rectangle(target, tl, br, (0, 0, 255), 2)
cv2.imwrite(self.out, target)
return tl[0]
def discern(self):
img1 = self.clear_white(self.gap)
img1 = cv2.cvtColor(img1, cv2.COLOR_RGB2GRAY)
slide = self.image_edge_detection(img1)
back = cv2.imread(self.bg, 0)
back = self.image_edge_detection(back)
slide_pic = cv2.cvtColor(slide, cv2.COLOR_GRAY2RGB)
back_pic = cv2.cvtColor(back, cv2.COLOR_GRAY2RGB)
x = self.template_match(slide_pic, back_pic)
# print('x: ', x)
# 输出横坐标, 即 滑块在图片上的位置
return x
def get_track(distance):
"""
根据偏移量获取移动轨迹
:param distance: 偏移量
:return: 移动轨迹
"""
# 移动轨迹
track = []
# 当前位移
current = 0
# 减速阈值
mid = distance * 3 / 4
# 计算间隔
t = 1
# 初速度
v = 0
while current < distance:
if current < mid:
# 加速度为正 2
a = 2
else:
# 加速度为负 3
a = -3
a = 3 if current < mid / 2 else (4 if current < mid else -4)
# 初速度 v0
v0 = v
# 当前速度 v = v0 + at
v = v0 + a * t
move = v0 * t + 1 / 2 * a * t * t
# 当前位移
current += move
# 加入轨迹
track.append(round(move))
# print '轨迹 ', track
return track
class TIKTOK:
def __init__(self,which):
#重置时先改变fp
'''
result = os.popen('python ' + cur_path + '\\' + 'tiktok_up_diy_key.py "{}"'.format(cookie_path)).read()
cookies = eval(result)
cookies_exe_dict = {}
for cookie in cookies:
cookies_exe_dict[cookie['name']] = cookie['value']
print(cookies_exe_dict)
msToken = cookies_exe_dict['msToken']
print(msToken)
s_v_web_id = cookies_exe_dict['s_v_web_id']
print(s_v_web_id)
exit(1)
'''
cur_path = os.path.abspath(os.path.dirname(__file__))
sys.path.append(cur_path)
chrome_options = webdriver.ChromeOptions()
# chrome_options.add_argument('--proxy-server={0}'.format(self.proxy.proxy))
# chrome_options.add_argument('--blink-settings=imagesEnabled=false')
chrome_options.add_argument("--disable-blink-features=AutomationControlled") # 禁用启用Blink运行时的功能
chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"]) # 去除浏览器检测框
chrome_options.add_argument('--user-data-dir='+cookie_path)
self.driver = webdriver.Chrome(chrome_options=chrome_options,
executable_path=get_chromedriver_path)
with open(f'{cur_path}'+'/stealth.min.js', encoding='utf-8') as f:
js = f.read()
self.driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
"source": js
})
self.curr_status()
print("wph睡眠2秒来输入验证码")
#time.sleep(300)
#base_url = "https://相应网址/detail/index.html?id=3602005992881188284"
##打印cookie 并携带cookie
base_url = "https://相应网址/detail/index.html?id=3610351231685539908"
self.driver.get(base_url)
self.driver.maximize_window()
time.sleep(2)
self.driver.refresh()
time.sleep(2)
for i in range(0, 50):
try:
print(f"-----------------{i}->")
bg = self.driver.find_element(By.ID, "captcha-verify-image").get_attribute('src')
print("bg------------->" + bg)
gap = ""
try:
gap = self.driver.find_element(By.XPATH, '//*[@id="captcha_container"]/div/div[2]/img[2]').get_attribute('src')
except Exception as e:
#path = os.path.join(os.getcwd(), 'selenium_chrome', 'rebootRouter.exe')
#anti = " ".join([path, ' '])
#anti = os.popen(anti).read()
time.sleep(100)
my_a = int(input('input number: '))
if my_a <1:
time.sleep(6000)
break
#exit(1)
print("gap------------->" + gap)
slider = Douyin_Slider(bg=bg, gap=gap)
distance = slider.discern()
print('移动距离:', distance)
slider = self.driver.find_element(By.XPATH, '//div[@id="secsdk-captcha-drag-wrapper"]/div[2]')
ActionChains(self.driver).click_and_hold(slider).perform()
_tracks = get_track(distance)
print('开始验证....')
print(_tracks)
new_1 = _tracks[-1] - (sum(_tracks) - distance)
_tracks.pop()
_tracks.append(new_1)
# 四个节点合并一个节点记得 节点总和-3
get_1 = _tracks[-1]
get_2 = _tracks[-2]
get_3 = _tracks[-3]
get_4 = _tracks[-4]
get_two = get_1 + get_2 + get_3 + get_4 - 7
_tracks.pop()
_tracks.pop()
_tracks.pop()
_tracks.pop()
_tracks.append(get_two)
_tracks.append(3)
_tracks.append(4)
# python让数据只有八个节点 [2, 4, 8, 10, 14, 17, 21, 21, 17] 变成[2, 4, 8, 10, 14, 54, 3, 4]
#省略一段核心,因为太敏感,可以q258599831或者私信大家一起学习
ActionChains(self.driver).release().perform()
ActionChains(self.driver).click(slider)
time.sleep(0.2)
# break
except Exception as e:
print(f"--------------22---{i}->")
break
if '初始化失败' in self.driver.page_source:
print(f"商详数据初始化失败")
exit(1)
cookies = self.driver.get_cookies()
jsonCookies = json.dumps(cookies)
with open('s_v_web_id.json', 'w') as f:
f.write(jsonCookies)
cookie_dict = {}
for cookie in cookies:
k = cookie['name']
v = cookie['value']
cookie_dict[k] = v
# print(cookie_dict)
# time.sleep(2)
cookie_dict = cookie_dict.get('s_v_web_id')
print("----------------------cookie_dict--->" + cookie_dict)
self.driver.refresh()
self.driver.close()
self.driver.quit()
##进行数据采集
updata = UpDataSlide(1, 2)
time.sleep(10)
TIKTOK(1)
4)逆向工作是辛苦的,反爬的道路是艰难的,就是有了大家砥砺前行,才有筑造了中国的互联网长城固若金汤,才让中国互联网能引领世界。建议大家好好学习逆向。
5)注意事项,大家通过学习逆向帮助中国互联网我很开心,切记不能攻击别人网站,也不能损害互联网安全,坚决要遵守互联网规则,用自己的聪明才智报销祖国。
6)欢迎大家私信共同进步。