获取山东省采购网招标/投标信息链接

from bs4 import BeautifulSoup
from urllib.request import urlopen
from urllib.parse import quote
import urllib
from urllib import request
from time import sleep
import requests
'''
山东省采购网
'''
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0'}

#遍历当前页面中的中标/招标信息
def getAllBiddingLinks(siteUrl, matchStr):
    opener = urllib.request.build_opener(urllib.request.HTTPHandler)
    urllib.request.install_opener(opener)
    
    quoteUrl = quote(siteUrl, safe='/:?=&#')
    #发送网络请求，如果连接失败，延时5秒，无限重试链接
    success = False
    while(success == False):
        try:
            #发送网络请求
            req = request.Request(quoteUrl,headers=headers,)
            response = urlopen(req)
        except requests.exceptions.ConnectionError as e:
            sleep(5)
        except HTTPError as e:
            sleep(5)
        else:
            success = True
    buff = response.read()
    html = buff.decode()
    bsObj = BeautifulSoup(html,"html.parser")
    for a in bsObj.findAll("a","aa"):
        title = a.attrs["title"] #中标/招标公告
        href = a.attrs["href"]
        if(matchStr.decode() in title):
            bidding_url = "http://www.ccgp-shandong.gov.cn/" + href[1:] 
            print(bidding_url)
            #发送网络请求，如果连接失败，延时5秒，无限重试链接
            success = False
            while(success == False):
                try:
                    #发送网络请求
                    bidding_req = request.Request(bidding_url,headers=headers,)
                    bidding_response = urlopen(bidding_req)
                except requests.exceptions.ConnectionError as e:
                    sleep(5)
                else:
                    success = True
            bidding_buff = bidding_response.read()
            bidding_html = bidding_buff.decode()

#b'\xe6\x8b\x9b\xe6\xa0\x87\xe5\x85\xac\xe5\x91\x8a'="招标公告".encode()
matchStr = b'\xe6\x8b\x9b\xe6\xa0\x87\xe5\x85\xac\xe5\x91\x8a'
#省采购公告
getAllBiddingLinks("http://www.ccgp-shandong.gov.cn/sdgp2017/site/channelall.jsp?colcode=0301", matchStr)
#市县采购公告
getAllBiddingLinks("http://www.ccgp-shandong.gov.cn/sdgp2017/site/channelall.jsp?colcode=0303", matchStr)

#b'\xe4\xb8\xad\xe6\xa0\x87\xe5\x85\xac\xe5\x91\x8a'="中标公告".encode()
matchStr = b'\xe4\xb8\xad\xe6\xa0\x87\xe5\x85\xac\xe5\x91\x8a'
#省中标公告
getAllBiddingLinks("http://www.ccgp-shandong.gov.cn/sdgp2017/site/channelall.jsp?colcode=0302", matchStr)
#市县中标公告
getAllBiddingLinks("http://www.ccgp-shandong.gov.cn/sdgp2017/site/channelall.jsp?colcode=0304", matchStr)

 

转载于:https://my.oschina.net/sitsit/blog/3052460