爬取动态的网页。

#coding=utf-8
#import webdriver from selenium
#要安装 pip install selenium
import re
#yum install nodejs   -->  npm install -g cnpm --registry=https://registry.npm.taobao.org
#npm install phantomjs-prebuilt@2.1.14 --ignore-scripts -> pip install phantomjs
from selenium import webdriver
# load PhantomJS driver
cur_url="http://bj.news.163.com/"
#在ubuntu  下 which phantomjs
driver = webdriver.PhantomJS('/usr/bin/phantomjs')
# set window size, better to fit the whole page in order to
# avoid dynamically loading data
driver.set_window_size(1280, 2400) # optional
# data page content
driver.get(cur_url)
# use page_source to get html content
content = driver.page_source
print content

# driver.find_element_by_class_name()

# data_time=re.findall('http:\/\/(.+\.){1,}163.com\/\d{2}\/\d{4}\/\d{2}\/.+\.html',content)
# print data_time


from bs4 import BeautifulSoup
soup = BeautifulSoup(content, 'lxml')
urls=[]
news_content=[]
for link in soup.select('div.na_detail  > div.news_title > h3 > a'):

    urls.append(link.get('href'))
    news_content.append(link.text)
print urls
for i in news_content:
       print i
print len(news_content)
print len(news_content)