几个python简单爬虫实例

# coding=utf-8

import requests
import re

header = {
          'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:47.0) Gecko/20100101 Firefox/47.0',
          'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
          'Accept-Language': 'en-US,en;q=0.5',
          'Accept-Encoding': 'gzip, deflate, br'
         }

url = "http://blog.csdn.net/d_pokemon?viewmode=list"

html = requests.get(url, headers=header)
html.encoding = html.apparent_encoding
print ('结果： ',html.status_code)
print ('原因： ',html.reason)

rule1 = 'title="阅读次数">阅读</a>(.*?)</span>'
rule2 = '<span class="link_title"><a href=".*?">(.*?)</a></span>'
patten1 = re.compile(rule1, re.S)
patten2 = re.compile(rule2, re.S)

tileArray = re.findall(patten2,html.content)
timeArray = re.findall(patten1,html.content)
#print tileArray
#print timeArray
i = 0
for tile in tileArray:
    print(tile+": "+timeArray[i])
    i = i+1

# coding=utf-8

from bs4 import BeautifulSoup
import urllib.request
import urllib


def getHtml(url, headers={}):
    req = urllib.request.Request(url, headers=headers)
    response = urllib.request.urlopen(req)
    content = response.read().decode('utf-8')
    response.close()
    return content

def parsehtmlMusicList(html):
    soup = BeautifulSoup(html, 'lxml')
    list_pic = soup.select('ul#m-pl-container li div img')
    list_nameUrl = soup.select('ul#m-pl-container li div a.msk')
    list_num = soup.select('div.bottom span.nb')
    list_author = soup.select('ul#m-pl-container li p a')
    n = 0
    length = len(list_pic)
    while n < length:
        print('歌单图片：' + list_pic[n]['src'] + '\n\n')
        print('歌单名称：' + list_nameUrl[n]['title'] + '\n\n歌单地址：' + list_nameUrl[n]['href'] + '\n\n')
        print('歌单播放量：' + list_num[n].text + '\n\n')
        print('歌单作者：' + list_author[n]['title'] + '\n\n作者主页：' + list_author[n]['href'] + '\n\n\n')
        n += 1


url = "http://music.163.com/discover/playlist"
html = getHtml(url, headers={
    'User-Agent': 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)',
    'Host': 'music.163.com'
})
parsehtmlMusicList(html)

这段代码是在网上找的，简单易懂，本来我要改用urllib2，后来发现python3.x没有urllib2这个模块，，，，python 3.x中urllib库和urilib2库合并成了urllib库。 其中urllib2.urlopen()变成了urllib.request.urlopen() urllib2.Request()变成了urllib.request.Request()