爬取新闻并存入excel中

from bs4 import BeautifulSoup
from lxml import etree
import requests
from splinter import Browser
from selenium import webdriver
import urllib
import time
import os
import re
import base64
from splinter import Browser
import sys
import urllib.request
import os
import xlwt
import sqlite3
#http://www.gongyibaodao.com/zyff/16490.html
#http://www.gongyibaodao.com/zyff/16545.html
headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36 Edg/89.0.774.57"
           }

#<div id="article">
#<h1 id="articleTitle">
#<div id="articleText">  all_p


def main():
    datalist = getData()
    #爬取网页
    savepath = "公益新闻.xls"
    saveData(datalist, savepath)
    #保存数据

#爬取网页
def getData():
    
    datalist = []
    num = 0
    for i in range(16491,16506):
        data = []
        url = ['http://www.gongyibaodao.com/zyff/{}.html'.format(i)]
        url = url[0]
        time.sleep(3)
        html = urllib.request.urlopen(url).read()
        #逐一解析数据
        soup = BeautifulSoup(html,"html.parser")
        middle_headline = soup.find('div', {'id': 'article'})
        save_headline_name = (middle_headline.find('h1')).get_text()
        #将标题存入data的第一列
        data.append(save_headline_name)
        middle_text = soup.find('div', {'id': 'articleText'})
        text_articles = middle_text.find_all('p')
        text_articles.encoding = 'utf-8'
        #将正文内容存入data的第二列中
        for item in text_articles:
            data.append(item.get_text())
        #将第一个新闻的标题和内容存入第一行
        datalist.append(data)
    #返回一个data列表
    return datalist

def saveData(datalist, savepath):
    # 把处理好的一个电影信息存储到datalist中
    print("save....")
    book = xlwt.Workbook(encoding="utf-8", style_compression=0)
    sheet = book.add_sheet('公益新闻', cell_overwrite_ok=True)
    col = ("标题", "内容")
    #range(0,2)指的是总共表头有几列
    for i in range(0, 2):
        sheet.write(0, i, col[i])
    #range(0,15)表示datalist总共有几行
    for i in range(0, 15):
        print("第%d条" % (i + 1))
        data = datalist[i]
        #range(0,2)与上面的range（0，2）含义相似
        for j in range(0, 2):
            sheet.write(i + 1, j, data[j])
    book.save('公益新闻.xls')

main()
print("全部存储")

//将csv格式转变为json格式
import csv
import json

csvfile = open(‘F:\公益新闻1.csv’,‘r’)

jsonfile =open(‘F:\公益新闻.json’,‘w’)

filenames=()

reader = csv.DictReader(csvfile,filenames)
for line in reader:
#json.dump(row,jsonfile)
json.dump(line, jsonfile)
jsonfile.write("\n")

import csv
import json

csvfile = open('F:\公益新闻1.csv','r')

jsonfile =open('F:\公益新闻.json','w')


filenames=()

reader = csv.DictReader(csvfile,filenames)
for line in reader:
    #json.dump(row,jsonfile)
    json.dump(line, jsonfile)
    jsonfile.write("\n")