python requests 爬取数据

python requests 爬取数据

import requests
from lxml import etree
import time
import pymysql
import json
headers={

    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36',
    'Content-Type':'application/x-www-form-urlencoded',
    'Pragma':'no-cache',
    'Upgrade-Insecure-Requests':'1',
    'Content-Length':'86',
    'Host':'www.bjda.gov.cn'
}

headers_xiangqing={

    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36',
    'Pragma':'no-cache',
    'Upgrade-Insecure-Requests':'1',
    'Host':'www.bjda.gov.cn'
}

dd={
    'pageSize':'20'
}

temp=[]

dd['currentPage'] = '10'
print(dd)
response = requests.post('http://www.bjda.gov.cn/eportal/ui?pageId=348736', headers=headers, data=dd)
selector = etree.HTML(response.text)
item_spider = list(set(selector.xpath('//tr[@class="chaxun_con"]//a/@href')))
temp.extend(item_spider)

for i in temp:
    print('http://www.bjda.gov.cn/eportal/ui?pageId=348738&'+i[1:])
    response=requests.get('http://www.bjda.gov.cn/eportal/ui?pageId=348738&'+i[1:],headers=headers_xiangqing)
    print(response.status_code)
    selector=etree.HTML(response.text)
    tr=selector.xpath('//table[@class="table_sjcx"]//tr')
    print(tr

 

posted on 2018-04-18 11:15 秦瑞It行程实录 阅读( ...) 评论( ...) 编辑 收藏

转载于:https://www.cnblogs.com/ruiy/p/8872962.html