维普页面信息抓取

#!/usr/bin/python
#encoding=utf-8
__author__ = 'Administrator'
from  bs4 import  BeautifulSoup
from prettytable import PrettyTable
import selenium
import sys
import urllib
import requests
import time
import re
import  csv
if __name__ == "__main__":
    import os
    from selenium import webdriver
    from selenium.webdriver.support.ui import WebDriverWait

    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0'}

    chromedriver = "/home/henson/Documents/pycharm/webdriver/chromedriver"
    os.environ["webdriver.chrome.driver"] = chromedriver
    driver = webdriver.Chrome(chromedriver)
    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
    driver.get('http://qikan.cqvip.com/zk/search.aspx?key=M%3d%E5%A4%A7%E6%B0%94%E6%B1%A1%E6%9F%93%E9%98%B2%E6%B2%BB&page=1&ids=&size=50#search-result-list')
    #inputElement = driver.find_element_by_xpath("//*[@id='b_Text0']")#//*[@id="b_Text0"]
    urlList=[]
    #title = driver.find_element_by_xpath('//*[@id="body"]/div/div[3]/div[4]/dl[1]/dt/a').text
    #print(title)


    f = open("/home/henson/Desktop/001/758.csv", "a+", encoding='utf-8')
    writer = csv.writer(f)
    writer.writerow(["title", "author", "cited", "fund"])
    x = PrettyTable(["number", "title", "author", "fund"])
    for i in  range(1,300):
        for j in range(1,51):
            title = driver.find_element_by_xpath(
                '//*[@id="body"]/div/div[3]/div[4]/dl['+str(j)+']/dt/a').text
            author = driver.find_element_by_xpath(
                '//*[@id="body"]/div/div[3]/div[4]/dl['+str(j)+']/dd[3]/span[@class="writer"]').text
            cited = driver.find_element_by_xpath(
                '//*[@id="body"]/div/div[3]/div[4]/dl['+ str(j)+']/dt/span[@class="cited"]').text
            print(title)
            print(author)
            print(cited)
            try:
                fund = driver.find_element_by_xpath(
                    '//*[@id="body"]/div/div[3]/div[4]/dl[' + str(j) + ']/dd[4]/span[@class="fund"]').text
                print(fund)
            except Exception:
                try:
                    title = driver.find_element_by_xpath(
                        '//*[@id="body"]/div/div[3]/div[4]/dl[' + str(j) + ']/dt/a').text
                    author = driver.find_element_by_xpath(
                        '//*[@id="body"]/div/div[3]/div[4]/dl[' + str(j) + ']/dd[3]/span[@class="writer"]').text
                    cited = driver.find_element_by_xpath(
                        '//*[@id="body"]/div/div[3]/div[4]/dl[' + str(j) + ']/dt/span[@class="cited"]').text
                    fund=' '
                    continue;
                except Exception:
                    break;
                break;
            #writer.writerows([title.replace(',',''),author.replace(',',''),cited.replace(',',''),fund.replace(',','')])  #写入CSV
            data = []
            #data.append((title, author, cited, fund))
            data.append((title,'+', author,'+', cited,'+' ,fund))   #通过+或者其他符号通过xls分列
            writer.writerows(data)




                #print(title)
                #print(author)
                #print(fund)
                #data={number:[title,author,fund]}
                #data=number+" "+title+" "+author.replace("\n", " ")+" "+fund.replace("\n", " ")
               # print('%-4s%-60s%-30s%-100s' %(number,title,author.replace("\n", " "),fund.replace("\n", " ")))
                #print(data)
                #print({number},{title},{author.replace("\n", " ")},{fund.replace("\n", " ")})
                #f=open('/home/henson/Documents/coding/pachong/plus.csv','wb')
                #f = open("/home/henson/Documents/coding/pachong/test.csv", "a+")
                #f = open("/home/henson/Documents/coding/pachong/test.txt", "a+",encoding='utf-8')
                #writer = csv.writer(f)
                #writer.writerow(["number", "title", "author", "fund"])

                #number=number.encode("utf-8")
                #title = title.encode("utf-8")
                #author = title.author("utf-8")
                #fund = fund.author("utf-8")
                #用replace把逗号替换掉

                    # 先写入columns_name

                    # 写入多行用writerows
                #writer.writerows([number,title,author,fund])
                #f.close()


            x.border = False
            x.align["number"] = "l"
            x.valign["title"] = "t"

            x.padding_width = 10
            x.add_row([title, author.replace("\n", " "), cited, fund.replace("\n", " ")])
            if j % 10 == 0:
                print(x)  # 迭代次数 每个多少输出一次？


            def myAlign(string, length):
                if length == 0:
                    return string
                slen = len(string)
                re = string
                if isinstance(string, str):
                    placeholder = ' '
                else:
                    placeholder = u'　'
                while slen < length:
                    re += placeholder
                    slen += 1
                return re



                #print(number,myAlign(title,60)+str+myAlign(author.replace("\n", " "),120),fund.replace("\n", " "))


        now_handle = driver.current_window_handle  # 获取当前窗口句柄
        all_handles = driver.window_handles  # 获取所有窗口句柄
        driver.find_element_by_xpath(
            '//*[@id="body"]/div/div[3]/div[6]/div[2]/span[2]/a[2]').click()  # 下一页
        time.sleep(2)
        for handle in all_handles:
            if (handle != now_handle):
                driver.switch_to_window(handle)
                # print("new web"+ driver.current_url)
    f.close()

data = []
#data.append((title, author, cited, fund))
data.append((title,’+’, author,’+’, cited,’+’ ,fund)) #通过+或者其他符号通过xls分列
writer.writerows(data)