Python最简单的网页抓取

1. Install 2 python packages:

$ sudo pip install requests
$ sudo easy_install beautifulsoup4

2. Creat test.py

#coding=utf-8

import requests

from bs4 import BeautifulSoup

#get url

def get_html( url):

response = requests.get(url)

response.encoding = 'utf-8'

return response.text

#get title

def get_title( html):

soup = BeautifulSoup(html, 'html.parser')

soup.select( 'p')[ 0].get_text()

title_content = soup.select( 'title')[ 0].get_text()

return title_content

#get text

def print_p( html):

soup = BeautifulSoup(html, 'html.parser')

for p in soup.select( 'p'):

print p.get_text()

url = "http://www.cityu.edu.hk/"

html = get_html(url)

title_content = get_title(html)

print title_content

print_p(html)

3.Go to folder of test.py then execute

$ python test.py

4. Output