百度舆情

import requests
import json
import codecs
import time
import random
import csv
from fake_useragent import UserAgent
import pandas as pd
from selenium import webdriver
from lxml import etree
from bs4 import BeautifulSoup as bs
from selenium import webdriver
from selenium.webdriver.common.proxy import Proxy
from selenium.webdriver.common.proxy import ProxyType
from selenium.webdriver.common.keys import Keys
def save_to_csv(p_title,p_source,p_label,attributes,createtime,similar,p_content):
    row = [p_title,p_source,p_label,attributes,createtime,similar,p_content]
    with open(r'D:\update\baiduyuqing.csv', 'a', newline='', encoding='utf-8') as file:
        f = csv.writer(file)        
        f.writerow(row)
def parse_detail(parse_data):
    soup=bs(parse_data,'lxml')
#    selector = etree.HTML(parse_data)

    d = soup.find_all('tr', class_="item J_titleTr")
    b = soup.find_all('tr', class_="J_itemSum item-sum")
    aa=0
#    a=0
    for content in d:
        aa+=1
        #//td[@class="title"]//a//text()
        print('第---%d---条信息'%aa)
        selector = etree.HTML(parse_data)
        p_title=content.find('td',class_="title").get_text()
        p_source=content.find_all('td',class_="wrap-active")[0].get_text()
        p_label=content.find_all('td',class_="wrap-active")[1].get_text()
        print(p_title)
#        correlation=''
#        label = "新闻"
#        createtime=content.find_all('td')[4].get_text()
        i = aa-1
#        p_content = selector.xpath('//tr[@class="J_itemSum item-sum show"]//p//text()')

        times=selector.xpath('//tr[@class="item J_titleTr"]//td[5]//text()')
        createtime = times[i]
        print(createtime)

#        attributes=content.find_all('td')[3].get_text()
        attribute=selector.xpath('//tr[@class="item J_titleTr"]//td[4]//text()')
        attributes = attribute[i]
        print(attributes)
#        similar = ''
#        similar=content.find_all('td',class_="J_similarBtn similar-btn").get_text()
        similars=selector.xpath('//*[@id="J_tBody"]//td[7]/a//text()')
        similar = similars[i]
        print(similar)
#        contents = selector.xpath('')
        i = aa-1
        p_content = b[i]
#         .split("</i>")[1]
        save_to_csv(p_title,p_source,p_label,attributes,createtime,similar,p_content)
file1=open('ip.txt','r')
ips=file1.readlines()

def get_ip():
    ip=ips[random.randint(1,500)]
    print(ip)
    proxy = Proxy(
        {
            'proxyType': ProxyType.MANUAL,
            'httpProxy': ip
        }
    )
    return proxy

def get_html(url,username,password):
    driver = webdriver.Chrome()
    driver.maximize_window()
    driver.get(url)
    driver.find_element_by_xpath('//a[@class="entry-login J_login di-header__action-item"]').click()  # 获取登陆标签并点击
    time.sleep(random.randint(3, 9))
    driver.find_element_by_xpath ('//input[@id="TANGRAM__PSP_4__userName"]').clear ()
    driver.find_element_by_xpath ('//input[@id="TANGRAM__PSP_4__userName"]').send_keys(username)
    time.sleep(random.randint(3, 9))
    driver.find_element_by_xpath('//input[@id="TANGRAM__PSP_4__password"]').clear()
    driver.find_element_by_xpath('//input[@id="TANGRAM__PSP_4__password"]').send_keys(password)
    driver.find_element_by_xpath('//input[@id="TANGRAM__PSP_4__submit"]').click()  # 获取登陆标签并点击
    time.sleep(random.randint(3,9))
#    driver.find_element_by_xpath('//input[@type="text"]').clear()
#    driver.find_element_by_xpath('//input[@type="text"]').send_keys("一带一路")
#    driver.find_element_by_xpath('//input[@type="text"]').send_keys(Keys.ENTER)
    driver.find_element_by_xpath('//input[@ip="J_search"]').clear()
    driver.find_element_by_xpath('//input[@ip="J_search"]').send_keys("一带一路")
    driver.find_element_by_xpath('//input[@ip="J_search"]').send_keys(Keys.ENTER)    
    time.sleep(random.randint(4,6))
    #a=driver.find_element_by_xpath('//td[@colspan="8"]')
    data=driver.page_source
    if data:
        parse_detail(data)
    page=1
    while True:
        page+=1
        print('page is--------',page)
        time.sleep(random.randint(3,9))
        try:
            driver.find_element_by_xpath('//a[@class="page-turn"]').click()  # 获取下一页标签并点击
            time.sleep(random.uniform(4.1,10.6))
            next_data = driver.page_source
            parse_detail(next_data)
        except:
            break
if __name__=='__main__':
    username = '13073706991'
    password = 'muya199529'
    url = "http://yuqing.baidu.com/?castk=LTE%3D"
#    proxy=get_ip()
    get_html(url,username,password)