带user agent访问网页取值

# -*- coding: utf-8 -*-
from urllib import request
from lxml import etree


url = 'http://******.com/'
head = {'User-Agent： 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0'}
req = request.Request(url, headers=head)
response = request.urlopen(req)
html = response.read()
html = etree.HTML(html)
new_url = html.xpath('//*[@class="lbcon"]/ul/li/a/@href')

print(new_url)

本文参考：http://blog.csdn.net/qq_34209701/article/details/72627603