from selenium import webdriver import time driver=webdriver.Chrome() try: driver.implicitly_wait(5) driver.get('https://doc.scrapy.org/en/latest/_static/selectors-sample1.html') #通过Xpath查 html = driver.find_element_by_xpath('/html') # html = driver.find_element_by_xpath('/html') #报错 print(html.tag_name) time.sleep(10) #//从根节点开始查找任意一个节点 div=driver.find_element_by_xpath('//div') print(div.tag_name) #@ #查找id为images的div节点 div = driver.find_element_by_xpath('//div[@id="images"]') print(div.tag_name) print(div.text) # #找到一个a节点 a = div.find_element_by_xpath('//a') print(a.tag_name) # #找到所有a节点 a_s = div.find_element_by_xpath('//a') print(a_s) # # #找到第一个a节点的href属性 # get.attribute:获取节点中某个属性 a = div.find_element_by_xpath('//a').get_attribute('href') # print(a) finally: driver.close()
from selenium import webdriver import time driver=webdriver.Chrome() try: driver.implicitly_wait(5) driver.get('https://doc.scrapy.org/en/latest/_static/selectors-sample1.html') #通过Xpath查 html = driver.find_element_by_xpath('/html') # html = driver.find_element_by_xpath('/html') #报错 print(html.tag_name) time.sleep(10) #//从根节点开始查找任意一个节点 div=driver.find_element_by_xpath('//div') print(div.tag_name) #@ #查找id为images的div节点 div = driver.find_element_by_xpath('//div[@id="images"]') print(div.tag_name) print(div.text) # #找到一个a节点 a = div.find_element_by_xpath('//a') print(a.tag_name) # #找到所有a节点 a_s = div.find_element_by_xpath('//a') print(a_s) # # #找到第一个a节点的href属性 # get.attribute:获取节点中某个属性 a = div.find_element_by_xpath('//a').get_attribute('href') # print(a) finally: driver.close()
''' 获取cookies ''' from selenium import webdriver import time driver = webdriver.Chrome() try: driver.implicitly_wait(10) driver.get('https://www.zhihu.com/explore') print(driver.get_cookies()) time.sleep(10) finally: driver.close()
''' 选项卡(未写) ''' #选项卡管理：切换选项卡，有js的方式windows.open,有windows快捷键：ctrl+t等，最通用的就是js的方式 import time from selenium import webdriver browser=webdriver.Chrome() try: browser.get('https://www.baidu.com') browser.execute_script('window.open()') print(browser.window_handles) #获取所有的选项卡 browser.switch_to_window(browser.window_handles[1]) browser.get('https://www.taobao.com') time.sleep(10) browser.switch_to_window(browser.window_handles[0]) browser.get('https://www.sina.com.cn') browser.close()
''' ActionChas动作链 ''' from selenium import webdriver from selenium.webdriver import ActionChains import time driver = webdriver.Chrome() driver.implicitly_wait(10) driver.get('http://www.runoob.com/try/try.php?filename=jqueryui-api-droppable') try: driver.switch_to.frame('iframeResult') draggable = driver.find_element_by_id('draggable') droppable = driver.find_element_by_id('droppable') actions = ActionChains(driver) source = draggable.location['x'] target = droppable.location['x'] print(source,target) distance = target-source print(distance) ActionChains(driver).click_and_hold(draggable).perform() s=0 while s<distance: ActionChains(driver).move_by_offset(xoffset=2,yoffset=0).perform() s+=2 ActionChains(driver).release().perform() time.sleep(10) finally: driver.close()
''' 前进、后退 ''' from selenium import webdriver import time driver = webdriver.Chrome() try: driver.implicitly_wait(10) driver.get('https://www.jd.com/') driver.get('https://www.baidu.com/') driver.get('https://www.cnblogs.com/') time.sleep(2) # 回退操作 driver.back() time.sleep(1) # 前进操作 driver.forward() time.sleep(1) driver.back() time.sleep(10) finally: driver.close()
from selenium import webdriver from selenium.webdriver import ChromeOptions import time r''' 步骤: 1、打开文件的查看，显示隐藏文件 2、找到C:\Users\administortra\AppData\Local\Google\Chrome\User Data 删除Default文件 3、重新打开浏览器，并登陆百度账号 - 此时会创建一个新的Default缓存文件 ''' # 获取options对象，参数对象 options = ChromeOptions() # 获取cookies保存路径 # 'C:\Users\administortra\AppData\Local\Google\Chrome\User Data' profile_directory = r'--user-data-dir=C:\Users\85265\AppData\Local\Google\Chrome\User Data' # 添加用户信息目录 options.add_argument(profile_directory) # 把参数加载到当前驱动中 chrome_options默认参数，用来接收options对象 driver = webdriver.Chrome(chrome_options=options) try: driver.implicitly_wait(10) driver.get('https://www.baidu.com/') ''' BDUSS:***** ''' # 添加用户cookies信息 # name、value必须小写 driver.add_cookie({"name": "BDUSS", "value": "G9qSDlTSXI4eGkzb3lHcH1dHpPVWJEVjlieXdkaXBzbmhMbWw0QzlkSUFBQAAAAAAAAAAAEAAAB9YymlysfLrbChtcS5ysrCAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKVTCF2lUwhdS"}) # 刷新操作 driver.refresh() time.sleep(10) finally: driver.close()
1、爬取一页京东商品
from selenium import webdriver from selenium.webdriver.common.keys import Keys import time driver = webdriver.Chrome() try: driver.implicitly_wait(10) # 往京东主页发送请求 driver.get('http://www.jd.com/') # 输入商品名称并回车继续 input_tag = driver.find_element_by_id('key') input_tag.send_keys('华为手机') input_tag.send_keys(Keys.ENTER) # 查找所有div # good_div= driver.find_element_by_id('J_goodsList') good_list = driver.find_elements_by_class_name('gl-item') for good in good_list: # 根据属性选择器查找 good_url = good.find_element_by_css_selector('.p-img a').get_attribute('href') # 商品名称 good_name = good.find_element_by_css_selector('.p-name em').text.replace("\n","--") # 商品价格 good_price = good.find_element_by_class_name('p-price').text.replace("\n",":") # 评价人数 good_commit = good.find_element_by_class_name('p-commit').text.replace("\n"," ") # 商品商家 good_form = good.find_element_by_class_name('J_im_icon').text.replace("\n"," ") good_content=f''' 商品链接：{good_url} 商品名称：{good_name} 商品价格：{good_price} 评价人数：{good_commit} 商品商家：{good_form} \n ''' print(good_content) with open('jd.txt','a',encoding='utf-8')as f: f.write(good_content) time.sleep(20) finally: driver.close()
2、爬取多页商品信息
from selenium import webdriver from selenium.webdriver.common.keys import Keys import time def get_good(driver): try: js_code=''' window.scrollTo(0,5000); ''' driver.execute_script(js_code) time.sleep(2) # 查找所有div # good_div= driver.find_element_by_id('J_goodsList') good_list = driver.find_elements_by_class_name('gl-item') n=1 for good in good_list: # 根据属性选择器查找 # 商品链接 good_url = good.find_element_by_css_selector('.p-img a').get_attribute('href') # 商品名称 good_name = good.find_element_by_css_selector('.p-name em').text.replace("\n","--") # 商品价格 good_price = good.find_element_by_class_name('p-price').text.replace("\n",":") # 评价人数 good_commit = good.find_element_by_class_name('p-commit').text.replace("\n"," ") # 商品商家 good_form = good.find_element_by_class_name('J_im_icon').text.replace("\n"," ") good_content=f''' 商品链接：{good_url} 商品名称：{good_name} 商品价格：{good_price} 评价人数：{good_commit} 商品商家：{good_form} \n ''' print(good_content) with open('jd.txt','a',encoding='utf-8')as f: f.write(good_content) next_tag = driver.find_element_by_class_name('pn-next') next_tag.click() time.sleep(2) get_good(driver) time.sleep(10) finally: driver.close() if __name__ == '__main__': good_name = input('请输入爬取商品信息：').strip() driver = webdriver.Chrome() driver.implicitly_wait(10) # 往京东主页发送请求 driver.get('http://www.jd.com/') # 输入商品名称并回车继续 input_tag = driver.find_element_by_id('key') input_tag.send_keys(good_name) input_tag.send_keys(Keys.ENTER) time.sleep(2) get_good(driver)
破解滑动验证：
''' 破解极验滑动验证 博客园登录url: https://account.cnblogs.com/signin?returnUrl=https%3A%2F%2Fwww.cnblogs.com%2F 1、输入用户名与密码，并点击登录 2、弹出滑动验证，获取有缺口与完整的图片 3、通过像素点进行比对，获取滑动位移距离 4、模拟人的行为轨迹 5、开始滑动 ''' from selenium import webdriver # 用来驱动浏览器的 from selenium.webdriver import ActionChains # 破解滑动验证码的时候用的 可以拖动图片 import time from PIL import Image # pip3 install pillow import random option = webdriver.ChromeOptions() option.add_argument('disable-infobars') driver = webdriver.Chrome(chrome_options=option) def get_snap(driver): # selenium自带的截图网页全屏图片 driver.save_screenshot('snap.png') img = driver.find_element_by_class_name('geetest_canvas_img') left = img.location['x'] upper = img.location['y'] right = left + img.size['width'] lower = upper + img.size['height'] # print(left, upper, right, lower) img_obj = Image.open('snap.png') # 对屏幕进行截取,获取滑动验证图片 image = img_obj.crop((left, upper, right, lower)) return image def get_image1(driver): time.sleep(0.2) js_code = ''' var x = document.getElementsByClassName('geetest_canvas_fullbg')[0].style.display="block"; console.log(x) ''' time.sleep(1) driver.execute_script(js_code) # 截取图片 img_obj = get_snap(driver) return img_obj def get_image2(driver): time.sleep(0.2) js_code = ''' var x = document.getElementsByClassName('geetest_canvas_fullbg')[0].style.display="none"; console.log(x) ''' driver.execute_script(js_code) time.sleep(1) # 截取图片 img_obj = get_snap(driver) return img_obj def get_distance(image1, image2): # 初始值 start = 60 # 滑块色差 color_num = 60 for x in range(start, image1.size[0]): for y in range(image1.size[1]): rgb1 = image1.load()[x, y] rgb2 = image2.load()[x, y] r = abs(rgb1[0] - rgb2[0]) g = abs(rgb1[1] - rgb2[1]) b = abs(rgb1[2] - rgb2[2]) if not (r < color_num and g < color_num and b < color_num): return x - 7 def get_stacks(distance): distance += 20 ''' 匀加速\减速运行 v = v0 + a * t 位移: s = v * t + 0.5 * a * (t**2) ''' # 初速度 v0 = 0 # 加减速度列表 a_list = [3, 4, 5] # 时间 t = 0.2 # 初始位置 s = 0 # 向前滑动轨迹 forward_stacks = [] mid = distance * 3 / 5 while s < distance: if s < mid: a = a_list[random.randint(0, 2)] else: a = -a_list[random.randint(0, 2)] v = v0 stack = v * t + 0.5 * a * (t ** 2) # 每次拿到的位移 stack = round(stack) s += stack v0 = v + a * t forward_stacks.append(stack) back_stacks = [-1, -1, -2, -3, -2, -3, -2, -2, -3, -1] return {'forward_stacks': forward_stacks, 'back_stacks': back_stacks} def main(): try: driver.get('https://passport.cnblogs.com/user/signin') driver.implicitly_wait(5) # 1.输入用户名与密码,点击登录 username = driver.find_element_by_id('LoginName') password = driver.find_element_by_id('Password') login_button = driver.find_element_by_class_name('ladda-label') time.sleep(1) username.send_keys('_tank_') time.sleep(1) password.send_keys('k46709394.') # 这里需要等待账号密码输入完毕后再点击登录按钮，否则的不弹框 time.sleep(1) login_button.click() # time.sleep(3) # 2.点击滑动验证按钮,获取图片 geetest_button = driver.find_element_by_class_name('geetest_slider_button') geetest_button.click() time.sleep(0.2) # 3.针对完整的图片进行截取 image1 = get_image1(driver) # 4.针对有缺口的图片进行截取 image2 = get_image2(driver) # 5.对比两张图片,获取滑动距离 distance = get_distance(image1, image2) # 6.模拟人为滑动轨迹 stacks = get_stacks(distance) # 7.根据滑动轨迹进行滑动 forward_stacks = stacks['forward_stacks'] back_stacks = stacks['back_stacks'] slider_button = driver.find_element_by_class_name('geetest_slider_button') time.sleep(0.2) ActionChains(driver).click_and_hold(slider_button).perform() time.sleep(0.2) for forward_stack in forward_stacks: ActionChains(driver).move_by_offset(xoffset=forward_stack, yoffset=0).perform() time.sleep(0.1) for back_stack in back_stacks: ActionChains(driver).move_by_offset(xoffset=back_stack, yoffset=0).perform() time.sleep(0.1) time.sleep(0.2) ActionChains(driver).move_by_offset(xoffset=5, yoffset=0).perform() ActionChains(driver).move_by_offset(xoffset=-5, yoffset=0).perform() ActionChains(driver).release().perform() time.sleep(50) finally: driver.close() if __name__ == '__main__': main()
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
