使用python抓取网站代码，并下载里面的url

方法一

#!/usr/bin/env python
#-*- coding:utf-8 -*-
#python2.7的代码

import os
import urllib,urllib2
from lxml import etree

rename_dic={}

header = {
    "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36",
    "Referer": "http://www.baidu.com/"
}
get_url = 'http://www.stats.gov.cn/tjsj/pcsj/rkpc/6rp/lefte.htm'
req = urllib2.Request(get_url,data=None,headers=header)
web_code = urllib2.urlopen(req)
content = web_code.read()

#通过xpath进行标签匹配
econtent = etree.HTML(content)
li_list = econtent.xpath("//ul/li/a")

for line in li_list:
    if 'xls' in line.attrib['href']:
        new_url ='http://www.stats.gov.cn/tjsj/pcsj/rkpc/6rp/%s' % line.attrib['href']
        name1 = '%s.xls' %line.text
        name = new_url.rsplit('/',1)[1]
        rename_dic[name] = name1
        print(new_url,name)
        #cmd = 'wget %s -O %s &> /dev/null' %(new_url,name)
        #os.system(cmd)
        #os.system('wget %s -O %s  &> /dev/null' %(new_url,name))  
        os.system('wget %s  &> /dev/null' %new_url)      
        #urllib.urlretrieve(new_url,name)   不要通过这种方式下载会有问题，里面的内容非法的URL地址访问

for oldname,newname in rename_dic.items():
    os.rename(oldname,newname)

效果图：

备注：

1、xpath 匹配语法
// 是代表所有的
/ 代表层级
[] 筛选
@ 筛选条件
2、 xpath 匹配到的内容我们又三个方法
tag 返回对象标签名称
attrib 返回对象属性
text 返回对象当中的文本

示例：
//div[@id='content']    匹配所有div标签下，id='content'的标签
//ul/li/a               匹配所有 ul 标签下的 li 标签下的 a 标签

通过urlretrieve方法获取到的文本内容：

方法二
使用BeautifulSoup处理html元素

#!/usr/bin/env python
#-*- coding:utf-8 -*-

import requests
from bs4 import BeautifulSoup
'''pip install bs4'''
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36",
    "Referer": "http://www.baidu.com/"
}

def get_url(url):
    web_data = requests.get(url=url, headers=headers)
    Soup = BeautifulSoup(web_data.content, 'lxml')
    data = Soup.find('ul')
    data = data.findAll('a')
    data_url = []
    data_name = []
    for item in data:
        data_url.append('http://www.stats.gov.cn/tjsj/pcsj/rkpc/6rp/{}'.format(item['href']))
        data_name.append(item.get_text())
    return data_url, data_name


def Download_data(url, name):
    for url_item, name_item in zip(url, name):
        print('正在下载{}。。。'.format(name_item))
        with open(r'data\{}.xls'.format(name_item), 'ab') as fl:
            fl.write(requests.get(url_item, headers=headers).content)

if __name__ == '__main__':
    url, name = get_url('http://www.stats.gov.cn/tjsj/pcsj/rkpc/6rp/lefte.htm')
    Download_data(url, name)

抓取房天下数据

#!/usr/bin/env python
#-*- coding:utf-8 -*-

import requests
from bs4 import BeautifulSoup
res = requests.get('http://land.fang.com/market/210100________1_0_1.html')
soup = BeautifulSoup(res.text,'html.parser')
l = []
for message in soup.select('.list28_text'):
    link = 'http://land.fang.com' + message.select('a')[0]['href']
    l.append(link)
for link in l:
    detail = requests.get(link)
    soup1 = BeautifulSoup(detail.text,'html.parser')
    messes=[]
    for mess in soup1.select('.banbox tr td'):
        messes.append(mess.text)
    print(messes[1:3])