python网上批量下载表格_Python批量下载网页中的表格存储到CSV文件中

'''

Created on Feb 28, 2017

@author: hcq908

'''

import csv

import os

# import re

from urllib.request import urlopen

from bs4 import BeautifulSoup

if __name__ == '__main__':

iCntTable = 0;

html = urlopen("https://en.wikipedia.org/wiki/Comparison_of_text_editors")

bsObj = BeautifulSoup(html, "html.parser")

oTables = bsObj.find_all("table")#选定第一个表格

for table in oTables:

iCntTable  =iCntTable + 1;

print('处理第%d个表格 \n'%iCntTable)

#获取表格名称

sTitleTag = table.find('caption');#标题只有一个

#     sMatchText = re.compile(r']+>', re.S)

#     sTextRemain = sMatchText.sub('', sTitleTag)

sTitleName=sTitleTag.get_text()+'.csv';

#路劲不存在是需要新建

sDir = 'files';

if not os.path.exists(sDir):

os.mkdir(sDir)

sCSV_Path = os.path.join(sDir,sTitleName);

rows = table.find_all("tr")#tr为每一行的标签

with open(sCSV_Path, 'wt', newline='', encoding='utf-8') as csvFile: #创建可写文件

writer = csv.writer(csvFile)

for row in rows:#先按行处理

csvRow = []#创建列表存储每行数据

for cell in row.find_all(['td', 'th']):#再按列处理,搜索一行中每格的内容,td或th都可以

csvRow.append(cell.get_text())#将每格中的数据追加到列表

writer.writerow(csvRow)#写入一行