python

from selenium import webdriver
import re
from bs4 import BeautifulSoup
import time
def url_open(driver,content):#爬取源代码
elem=driver.find_element_by_css_selector(’#q’)
elem.send_keys(content)
submit=driver.find_element_by_css_selector(’#J_TSearchForm > div.search-button > button’)
submit.click()
html=driver.page_source
return html
def url_xi(html):#变为B
soup = BeautifulSoup(html, ‘lxml’)
return soup
def url_page(soup):#页数
page_number=soup.find(‘div’,class_=‘total’)
number=page_number.get_text()
page=re.findall(r’\d+’,number)
for i in page:
for m in range(2,int(i)+1):
time.sleep(2)
input=driver.find_element_by_css_selector(’#mainsrp-pager > div > div > div > div.form > input’)
input.clear()
input.send_keys(m)
sure=driver.find_element_by_css_selector(’#mainsrp-pager > div > div > div > div.form > span.btn.J_Submit’)
sure.click()
html1=driver.page_source
soup1=url_xi(html1)
url_jie(soup1)

def url_jie(soup):#对源代码进行解析
res = re.compile(r’<img alt="(.?)" .?/>’, re.S)#爬取名称时所用的正则表达式
res1=re.compile(r’

(. ?)

’, re.S)#爬取销售情况
res2=re.compile(r’ (.?)’,re.S)
address=soup.find_all(‘div’,class_=‘location’)
title=soup.find_all(‘img’,class_=‘J_ItemPic img’)#名称
price=soup.find_all(‘div’,class_=‘price g_price g_price-highlight’)#价格
shop=soup.find_all(‘a’,class_=‘shopname J_MouseEneterLeave J_ShopInfo’)#店铺
list=[]
add1=[]
store=[]
for i in price:
# print(i.get_text())
list.append(i.get_text().replace(’\n’,’’))
for address1 in address:
add1.append(address1.get_text().replace(’\n’,’’))
for m in shop:
m = m.get_text()
store.append(m.replace(’\n’,’’))
sale = soup.find_all(‘div’, class_=‘deal-cnt’)#销售人数
place=soup.find_all(‘div’,class_=‘location’)
for title1,sale1,price1,list1,address1,store1 in zip(title,sale,price,list,add1,store):
data={
‘title1’: re.findall(res, str(title1)),
‘sale1’:re.findall(res1,str(sale1)),
# ‘price1’:price.get_text()
‘list1’:list1,
‘address1’:address1,
‘store1’:store1
}
print(data)

if name==‘main’:
url=“https://www.taobao.com/”
content = input(“请输入查询内容：”)
driver=webdriver.Firefox()
driver.get(url)
html=url_open(driver,content)
soup=url_xi(html)
url_jie(soup)
url_page(soup)
from selenium import webdriver
import re
from bs4 import BeautifulSoup
import time
def url_open(driver,content):#爬取源代码
elem=driver.find_element_by_css_selector(’#q’)
elem.send_keys(content)
submit=driver.find_element_by_css_selector(’#J_TSearchForm > div.search-button > button’)
submit.click()
html=driver.page_source
return html
def url_xi(html):#变为B
soup = BeautifulSoup(html, ‘lxml’)
return soup
def url_page(soup):#页数
page_number=soup.find(‘div’,class_=‘total’)
number=page_number.get_text()
page=re.findall(r’\d+’,number)
for i in page:
for m in range(2,int(i)+1):
time.sleep(2)
input=driver.find_element_by_css_selector(’#mainsrp-pager > div > div > div > div.form > input’)
input.clear()
input.send_keys(m)
sure=driver.find_element_by_css_selector(’#mainsrp-pager > div > div > div > div.form > span.btn.J_Submit’)
sure.click()
html1=driver.page_source
soup1=url_xi(html1)
url_jie(soup1)

def url_jie(soup):#对源代码进行解析
res = re.compile(r’<img alt="(.?)" .?/>’, re.S)#爬取名称时所用的正则表达式
res1=re.compile(r’

(. ?)

’, re.S)#爬取销售情况
res2=re.compile(r’ (.?)’,re.S)
address=soup.find_all(‘div’,class_=‘location’)
title=soup.find_all(‘img’,class_=‘J_ItemPic img’)#名称
price=soup.find_all(‘div’,class_=‘price g_price g_price-highlight’)#价格
shop=soup.find_all(‘a’,class_=‘shopname J_MouseEneterLeave J_ShopInfo’)#店铺
list=[]
add1=[]
store=[]
for i in price:
# print(i.get_text())
list.append(i.get_text().replace(’\n’,’’))
for address1 in address:
add1.append(address1.get_text().replace(’\n’,’’))
for m in shop:
m = m.get_text()
store.append(m.replace(’\n’,’’))
sale = soup.find_all(‘div’, class_=‘deal-cnt’)#销售人数
place=soup.find_all(‘div’,class_=‘location’)
for title1,sale1,price1,list1,address1,store1 in zip(title,sale,price,list,add1,store):
data={
‘title1’: re.findall(res, str(title1)),
‘sale1’:re.findall(res1,str(sale1)),
# ‘price1’:price.get_text()
‘list1’:list1,
‘address1’:address1,
‘store1’:store1
}
print(data)

if name==‘main’:
url=“https://www.taobao.com/”
content = input(“请输入查询内容：”)
driver=webdriver.Firefox()
driver.get(url)
html=url_open(driver,content)
soup=url_xi(html)
url_jie(soup)
url_page(soup)