【爬虫专栏8】pandas直接提取表格数据

长江学者数据

import pandas as pd
import csv

def get_one_page(num):
    url = 'http://news.sciencenet.cn/htmlnews/2018/1/399176.shtm'
    tb = pd.read_html(url, skiprows=[0])[num]  # 跳过前两行
    return tb # 去掉最后一行


with open(r'E:\vscode_code\爬虫测试\pandas提取\changjiang.csv', 'w', encoding='utf-8-sig', newline='') as f:
    csv.writer(f).writerow(['推荐学校', '姓名', '岗位名称', '现任职单位'])

for i in range(3):  # 目前116页数据
    get_one_page(i).to_csv(r'E:\vscode_code\爬虫测试\pandas提取\changjiang.csv', mode='a', encoding='utf_8_sig', header=0, index=0)
    print('第'+str(i+1)+'张表格抓取完成')

双色球数据

import pandas as pd
import csv

def get_one_page(page):
    url = 'http://kaijiang.zhcw.com/zhcw/html/ssq/list_%s.html' % (str(page))
    tb = pd.read_html(url, skiprows=[0, 1])[0]  # 跳过前两行
    return tb.drop([len(tb)-1])  # 去掉最后一行


with open(r'E:\vscode_code\爬虫测试\pandas提取\qiu1.csv', 'w', encoding='utf-8-sig', newline='') as f:
    csv.writer(f).writerow(['开奖日期', '期号', '中奖号码', '销售额(元)', '中奖注数一等奖', '中奖注数二等奖', '详细'])

for i in range(1,21):  # 目前116页数据，取20页
    get_one_page(i).to_csv(r'E:\vscode_code\爬虫测试\pandas提取\qiu1.csv', mode='a', encoding='utf_8_sig', header=0, index=0)
    print('第'+str(i)+'页抓取完成')