运行文件,可把相关变量,函数保存在当前,不必多次打开chrome
%run my_script.py
文件内容:
my_script.py
import pymongo,time,os,json,random
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from pyquery import PyQuery as pq
from config import * #数据库、关键词、爬取页数 配置信息
from urllib.parse import quote
# browser = webdriver.Chrome()
# browser = webdriver.PhantomJS(service_args=SERVICE_ARGS)
chrome_options = webdriver.ChromeOptions()
# chrome_options.add_argument('--headless')
chrome_options.add_argument('user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36')
browser = webdriver.Chrome(r'F:\Demo\python\dianshang\chromedriver.exe',chrome_options=chrome_options)
# 读取隐藏webdriver特征的js
with open('./stealth.min.js') as f:
js = f.read()
# 在每个页面加载前执行隐藏driver特征的js实现反爬
browser.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
"source": js
})
wait = WebDriverWait(browser, 20)
client = pymongo.MongoClient(MONGO_URL) #读取数据库连接地址
db = client[MONGO_DB] #读取数据库名
#浏览器复制到原始cookie文件
original_cookie_path='./cookies/original_cookies.txt'
cookies_path='./cookies/cookies.txt' #转化为json后的cookies
url='https://pinduoduo.com'
#把从浏览器复制的原始cookies转为json格式并保存
def original_to_cookies():
with open(original_cookie_path) as f:
original_cookie=f.read()
cookie_list=original_cookie.split(';')
# print(cookie)
cookies={}
for c in cookie_list:
k,v=c.split('=',1)
cookies[k]=v
# print(cookies)
with open (cookies_path,'w',encoding='utf-8') as f:
f.write(json.dumps(cookies))
print('cookie成功转为json格式并保存')
# 首次手动登录把cookie保存到文本
def save_cookies(cookies):
# 保存cookies到文件,参数来自browser.get_cookies()
path = os.getcwd() + '/cookies/'
if not os.path.exists(path):
os.makedirs(path)
with open(path + 'cookies.txt', 'w') as f:
cookie = json.dumps(cookies)
f.write(cookie)
print('cookie登录信息成功保存到文件')
def read_add_cookie():
#cookie登录:读取文件中的cookie并添加到浏览器中实现自动登录
path = os.getcwd() + '/cookies/'
if os.path.exists(path):
# 读取添加cookie
with open('./cookies/cookies.txt','r',encoding='utf-8') as f:
cookies=json.loads(f.read())
browser.get(url) #刷新使cookie生效
browser.delete_all_cookies()
for cookie in cookies:
browser.add_cookie(cookie)
browser.refresh() # 添加cookie后要刷新才能生效
print('cookies登录信息成功添加到浏览器')
else:
print('没有cookie文件!请进行初始手动登录!')
def is_Login():
# 是否登录:查找登录窗口的元素列表(注意是elements)
login=browser.find_elements(By.XPATH,'//*[@id="mf-mms-goods-container"]/div/div/div[1]/span')
if len(login)==1: #存在证明登录成功,返回真
return True
elif len(login)==0:
return False
def login():
if not is_Login():
print('cookie登录中,请稍等……')
read_add_cookie()
time.sleep(5)
# 如果没有登录则循环等待5秒
while not is_Login():
print('cookie登录失败,请手动进行登录')
time.sleep(5)
# 结束循环表示登录成功,把cookies保存到文件
print('手动登录成功!')
cookies=browser.get_cookies()
save_cookies(cookies) #保存登录成功的cookie以便下次登录
else:
print('已经登录,即将开始数据采集')