python爬取js动态网页_python爬取动态网页

1.首先下载phantomjs、selenium，将phantomjs放于设置环境变量的目录中，

2.尝试获取加载js后的单页面，

Paste_Image.png

from urllib import request

import urllib

from bs4 import BeautifulSoup as bs

import re

import os

import pandas as pd

import time

import random

from selenium import webdriver

def getHtml(url):

driver=webdriver.PhantomJS();

driver.get(url)

return driver.page_source

page_info=getHtml('http://tv.sohu.com/item/MTIwNjUzMg==.html')

soup = bs(page_info, 'html5lib')

tp=soup.find('em','total-play').string

Paste_Image.png

发现播放量成功得到了加载，爬取成功。

3.尝试多页面的爬取

由于爬取中需要模拟浏览器，加载js文件，因此如果未加载完全进行处理，就会获取不到信息。确保加载完全，需要设置等待时间，爬取速度较慢。将代码改写为多进程可加快速度，但仍然受到限制。

from urllib import request

import urllib

from bs4 import BeautifulSoup as bs

import re

import pandas as pd

import time

import random

import multiprocessing

from itertools import chain

from selenium import webdriver

import sys

sys.setrecursionlimit(10000000)

def urlAdd():

list_type=['1100','1101','1102','1103','1104','1105','1106','1107','1108','1109','1110','1111','1112','1113','1114'

,'1115','1116','1117','1118','1119','1120','1121','1122','1123','1124','1125','1127','1128']

list_loc=['1000','1001','1002','1003','1004','1015','1007','1006','1014']

list_time=['2017','2016','2015','2014','2013','2012','2011','2010','11','90','80','1']

return list_loc,list_type,list_time

def PageCreate():

<