代理和IP的使用
import requests
def get_ip():
response = requests.get('http://piping.mogumiao.com/proxy/api/get_ip_bs?appKey=775206edf3dc4329ba04568b75a66a30&count=4&expiryDate=0&format=2&newLine=3')
if response.text[0] == '{':
print('提取IP失败')
return None
return [x for x in response.text.split('\n') if x]
def get_net_data():
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36'
}
ips = get_ip()
if not ips:
print('ip获取失败,等10秒以后重新运行')
return
proxies = {
'http': ips[0],
'https': ips[1]
}
response = requests.get('https://movie.douban.com/top250', headers=headers, proxies=proxies)
print(response.text)
if __name__ == '__main__':
get_net_data()
使用代理和优化程序
import requests
import time
def get_ip():
response = requests.get('http://piping.mogumiao.com/proxy/api/get_ip_bs?appKey=775206edf3dc4329ba04568b75a66a30&count=4&expiryDate=0&format=2&newLine=3')
if response.text[0] == '{':
print('提取IP失败')
return None
return [x for x in response.text.split('\n') if x]
def get_net_data():
while True:
ips = get_ip()
if ips:
break
time.sleep(5)
print('ip获取成功:', ips)
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36'
}
proxies = {
'http': ips[0],
'https': ips[1]
}
url = 'https://movie.douban.com/top250'
response = requests.get(url, headers=headers, proxies=proxies)
print(response.text)
get_net_data()
selenium的基本功能
from selenium.webdriver import Chrome
b = Chrome()
b.get('https://www.jd.com/')
print(b.page_source)
selenium常规交互
from selenium.webdriver import Chrome
from selenium.webdriver.common.keys import Keys
import time
b = Chrome()
b.get('https://www.51job.com/')
search_input = b.find_element_by_css_selector('#kwdselectid')
search_input.send_keys('数据分析')
search_input.send_keys(Keys.ENTER)
print(b.page_source)
print('--------------------------------------------------------------------------------------------')
next = b.find_element_by_css_selector('.next')
next.click()
print('+++++++++++++++++')
time.sleep(1)
print(b.page_source)
selenium常用配置
from selenium.webdriver import Chrome, ChromeOptions
import requests
import time
def get_ip():
response = requests.get('http://piping.mogumiao.com/proxy/api/get_ip_bs?appKey=775206edf3dc4329ba04568b75a66a30&count=4&expiryDate=0&format=2&newLine=3')
if response.text[0] == '{':
print('提取IP失败')
return None
return [x for x in response.text.split('\n') if x]
while True:
ips = get_ip()
if ips:
break
time.sleep(1)
print(ips)
options = ChromeOptions()
options.add_experimental_option('excludeSwitches', ['enable-automation'])
options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images": 2})
b = Chrome(options=options)
b.get('https://movie.douban.com/top250')
print(b.page_source)
爬淘宝
from selenium.webdriver import Chrome, ChromeOptions
from selenium.webdriver.common.keys import Keys
options = ChromeOptions()
options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images": 2})
b = Chrome(options=options)
b.get('https://www.taobao.com')
cookies = eval(open('files/taobao.txt', encoding='utf-8').read())
for cookie in cookies:
if cookie['secure']:
b.add_cookie(cookie)
b.get('https://www.taobao.com')
search_input = b.find_element_by_id('q')
search_input.send_keys('鞋子')
search_input.send_keys(Keys.ENTER)
print(b.page_source)
获取和保存cookie值
from selenium.webdriver import Chrome
from selenium.webdriver.common.keys import Keys
import time
def save_cookie():
b = Chrome()
b.get('https://www.taobao.com')
search_input = b.find_element_by_id('q')
search_input.send_keys('鞋子')
search_input.send_keys(Keys.ENTER)
time.sleep(10)
cookies = b.get_cookies()
f = open('files/taobao.txt', 'w', encoding='utf-8')
f.write(str(cookies))
b.quit()
save_cookie()
爬51json数据分析
from selenium.webdriver import Chrome
from bs4 import BeautifulSoup
import csv
from selenium.webdriver.common.keys import Keys
import re
import time, json
f = open('files/数据分析.csv', 'a', encoding='utf-8')
writer = csv.writer(f)
writer.writerow(['岗位', '薪资', '公司', '地址'])
b = Chrome()
def get_net_data():
b.get('https://www.51job.com/')
search_input = b.find_element_by_css_selector('#kwdselectid')
search_input.send_keys('数据分析')
search_input.send_keys(Keys.ENTER)
while True:
page_div = b.find_element_by_css_selector('.rt.rt_page')
pages = page_div.text.split('/')
all_page = int(pages[-1])
current_page = int(pages[0])
save_data(b.page_source)
if current_page < 10:
next = b.find_element_by_css_selector('.next')
next.click()
else:
break
def save_data(html: str):
soup = BeautifulSoup(html, 'lxml')
all_job_div = soup.select('.j_joblist>.e')
one_page_jobs = []
for job_div in all_job_div:
name = job_div.select_one('.jname.at').get_text()
sal = job_div.select_one('.sal').get_text()
company = job_div.select_one('.cname.at').get_text()
job_url = job_div.select_one('.el').attrs['href']
one_page_jobs.append([name, sal, company, job_url])
writer.writerows(one_page_jobs)
if __name__ == '__main__':
get_net_data()