[python]抓取网页的内容

#-*- coding: UTF-8 -*- 

import urllib2, BeautifulSoup
# @param url: complete url
#             完整的url
# @param usr, pwd: if the page need account, 
#        \p usr and \p pwd will be used
#             当访问的页面需要密码的时候会用到
# @return: the formatted string content of the url
#             用了BeautifulSoup返回结果文本
def getWebPage(url, usr=None, pwd=None):
    if not usr and not pwd:
        content = urllib2.urlopen(url).read()
    else:
        pwdMgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
        pwdMgr.add_password(None, url, usr, pwd)
        handler = urllib2.HTTPBasicAuthHandler(pwdMgr)
        opener = urllib2.build_opener(handler)
        page = opener.open(url).read()
        content = BeautifulSoup.BeautifulSoup(page).prettify()
    return content

url='http://www.csdn.net/'
print getWebPage(url)