python会自动保存吗_python爬取文件自动保存本地-Go语言中文社区

趁热又把文件保存到本地的代码实现了。根据目标网站的文件分类形式

我选择先处理站点每5分钟的数据，以district->year->month的文件夹嵌套形式来存储文件。同时用类包装了上一篇爬虫的代码，代码分别如下：

AutoFile.py

import os

import urllib

from urllib.request import urlopen

import time

import random

class AutoFile:

root_path = "F:Pemsdata\"

head_url = 'http://pems.dot.ca.gov'

request = urllib.request

def __init__(self, filetype):

print("正在启动文件模块..........")

self.root_path = self.root_path + filetype + '\'

proxy = self.request.ProxyHandler({

'http': '127.0.0.1:1080',

'https': '127.0.0.1:1080'

})

opener = self.request.build_opener(proxy)

self.request.install_opener(opener)

print("正在加载目录：" + self.root_path)

def mkdir(self, path):

path = path.strip()

path = path.rstrip("\")

isExists = os.path.exists(path)

# 判断结果

if not isExists:

os.makedirs(path)

print(path + ' 创建成功')

else:

print(path + ' 目录已存在')

return True

def file_num(self, f_path):

os.chdir(f_path)

li = os.listdir()

num = 0

for i in li:

if i[len(i) - 2:len(i)] == 'gz':

num = num + 1

print("在" + f_path + "目录下已经存文件个数：" + str(num))

return num

def check_dir(self, f_district_path, f_yy_path, month, list_path):

f_path = self.root_path + f_district_path

if (self.mkdir(f_path)): # 判断该区域目录是否存在

f_path = f_path + '\' + f_yy_path

if (self.mkdir(f_path)): # 判断该区域下该年份目录是否存在

f_path = f_path + '\' + month

if (self.mkdir(f_path)): # 判断月份是否存在

print("开始下载........")

num = self.file_num(f_path)

self.download_file(f_path, list_path, num)

def schedule(self, a, b, c):

per = 100 * a * b / c

if per > 100:

per = 100

print('%.2f%%' % per)

def download_file(self, f_path, list_file, num):

e_num = len(list_file)

if (num == e_num): # 比较已经下载了多少个文件

print("已经是最新的数据")

else:

d_num = e_num - num

print("还需要下载文件个数：" + str(d_num))

d_num = num + d_num

for i in range(num, d_num, 1): # 从已经下载的文件开始，避免重复下载

day = list_file[i]

print("正在下载文件：" + day['file_name'])

time.sleep(random.randint(1, 3))

print(self.head_url + day['url'])

self.request.urlretrieve(self.head_url + day['url'],

f_path + '\' + day['file_name'], self.schedule)

print("下载结束")

AutoLogin.py 回调函数借鉴了网上的例子，但很奇怪我这里的文件总大小是-1 ?

import requests

import json

from BD2019.autoFile import AutoFile

class AutoLogin:

head_url = 'http://pems.dot.ca.gov'

district_id = '0'

yy = '0'

filetype = ''

session = ''

proxies = {

'http': '127.0.0.1:1080',

'https': '127.0.0.1:1080'

}

headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "

"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"}

data = {"redirect": "", "username": "账号",

"password": "密码", "login": "Login"}

autofile = None

def __init__(self):

print("正在启动爬虫模块..........")

self.session = requests.session()

self.session.post("http://pems.dot.ca.gov/?dnode=Clearinghouse",

proxies=self.proxies, data=self.data)

def get_station_5min_url(self, year, area_id):

if year < 2001 or year > 2019:

print("年份不合法！爬取失败！")

return None

file_url = self.head_url + "/?srq=clearinghouse&"

self.yy = str(year) # yy 年份 2001-2019 这里先以范围优先，故时间定位2019

self.filetype = 'station_5min' # type 数据内容的种类 这里先以5分钟站点检测数据为例

self.autofile = AutoFile(self.filetype) # 加载文件处理的类

for id in range(3, area_id, 1): # district_id 范围 3-12

self.district_id = str(id)

print('正在爬取' + self.yy + '年,第' + self.district_id +

'区域,类型为' + self.filetype + '的数据')

html = self.session.get(file_url +

'district_id=' + self.district_id +

'&geotag=null&yy=' + self.yy +

'&type=' + self.filetype +

'&returnformat=text',

proxies=self.proxies, headers=self.headers)

data = json.loads(html.text)

urls = data['data']

for month in urls: # 获取月份信息

print('正在爬取' + month + '的内容')

list_path = list()

for day in urls[month]: # 获取文件信息

list_path.append(day)

# print(day['file_name'] + ' ' + self.head_url + day['url'])

self.autofile.check_dir(self.district_id, self.yy, month, list_path)

还有一个main函数：

from BD2019.autoLogin import AutoLogin

if __name__ == '__main__':

superman = AutoLogin()

superman.get_station_5min_url(2019, 4)

# for year in range(2001, 2019, 1): # 轻易别尝试 (。﹏。)

# for district_id in range(3, 12, 1):

# superman.get_station_5min_url(year, district_id)

运行截图如下：

做到这里基本上5分钟站点的数据就可以爬取了。