python 简单爬虫原理

#coding=utf-8

 

#导入常用模块

import urllib

import urllib2

import cookielib

from bs4 import BeautifulSoup

 

#抓取的url

url = "http://www.baidu.com"

values = {

           'userName':'aaaaaa',

           'password':'bbbbbb'

         }

postdata = urllib.urlencode(values)

user_agent = "Mozilla/5.0 (Windows NT 6.1;WOW64)"

headers = {"User-Agent":user_agent}

request =urllib2.Request(url, data = None, headers=headers)

try:

    response =urllib2.urlopen(request, timeout = 2)

except urllib2.HTTPError, e:

    print e.code

except urllib2.URLError, e:

    print e.reason

except:

    print"Error"

  

data = response.read()

soup = BeautifulSoup(data, "lxml")

for link in soup.find_all('a'):

    print link