python抓资源_python抓网页资源小脚本

#!/usr/bin/env python

# coding: utf-8

import urllib

def filter_src(file_name):

resource_list = []

f_obj = open(file_name)

for f_line in f_obj:

if '404' in f_line:

str_goal = f_line.strip().split(' ')[7]

if not str_goal in resource_list:

print str_goal

if '/static' in str_goal:

str_goal = str_goal.replace('/static', '')

resource_list.append(str_goal[:-1])

print resource_list

return resource_list

def down_src(source_list):

base_url = "http://www.ttcrm.com"

down_path = r"src"

for source in source_list:

source_url = base_url + source

source_path = down_path + source

print source_url

source_stram = urllib.urlopen(source_url)

f_obj = open(source_path,'wb')

f_obj.write(source_stram.read())

if __name__=='__main__':

file_name = 'src.txt'

source_list = filter_src(file_name)

down_src(source_list)

关键点在于保存是以二进制方式保存！

f_obj = open(source_path,'wb')

f_obj.write(source_stram.read())