from selenium.webdriver.common.keys import Keys
from selenium import webdriver
import pandas as pd
import time
#driver = webdriver.Chrome()
driver = webdriver.Firefox()
def getPage(driver):
data=[]
driver.get("url")
#driver.page_source
element=driver.find_element_by_css_selector('html')
html=element.get_attribute('outerHTML')
#do sonme analyze
c=element.find_elements_by_css_selector(".jiedian")
index=0
for i in c:
if i.find_elements_by_css_selector(".ec-tuiguang") and i.find_elements_by_css_selector(".ec_title"):
t=i.find_element_by_css_selector(".ec_title h3")
u=i.find_element_by_css_selector("span.c-showurl")
item=[time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),str(index),t.text,u.text]
data.append(item)
print "/".join(item)
index+=1
return data
allData=[]
for j in range(0,10):
d=getPage(driver)
allData.extend(d)
time.sleep(10)
df=pd.DataFrame(allData,columns=['time','index','title','url'])
df.to_csv("data.csv",encoding='utf8',index=False)
driver.close()