【金融行业招标网站爬虫实现】

于 2024-10-12 11:16:55 发布
阅读量170 收藏
点赞数 1
文章标签： 金融 爬虫
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/weixin_45851870/article/details/142872738
版权

为方便公司内部销售员工更加快捷的方便获取到招标系统，开发了招标大师邮件提醒服务，本项目目前已经收集10+以上的招标系统，稳定运行一年，有需要的自取 

import requests
from bs4 import BeautifulSoup
import json
import datetime
import smtplib
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from email.header import Header
from jinja2 import Template
from dateutil import parser
import pytz  # 导入 pytz 模块
# from datetime import strptime
import sys
def get_dongfang(days):
    # 设置URL和请求头
    url = "https://supplier.dfzq.com.cn/suppliers/announcement/getAnnouncement"
    headers = {
        "sec-ch-ua": '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
        "sec-ch-ua-mobile": "?0",
        "Authorization": "",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
        "isOut": "supplier",
        "sessionId": "dfef71f6-24da-1e85-a952-d005332e4e20",
        "tgticketUrl": "aHR0cHM6Ly9zdXBwbGllci5kZnpxLmNvbS5jbi9zdXBwbGllcnMtcG9ydGFsL3BhZ2Uvc3VwcGxpZXIvYW5ub3VuY2VtZW50L2JpZGRpbmctbGlzdC5odG1s",
        "license": "telehot-inner",
        "sec-ch-ua-platform": '"Windows"',
        "Accept": "*/*",
        "Sec-Fetch-Site": "same-origin",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Dest": "empty",
        "Referer": "https://supplier.dfzq.com.cn/suppliers-portal/page/supplier/announcement/bidding-list.html",
        "Accept-Encoding": "gzip, deflate",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Cookie": "isload=true"
    }

    response = requests.get(url, headers=headers)

    # 检查请求是否成功
    if response.status_code == 200:
        # 将返回的JSON解析为Python的字典
        data = response.json()
        urls = {}
        for result in data["result"]["announcementList"]["content"]:
            publishTime_str = result['createTime']
            # print(publishTime_str)
            publishTime = parser.parse(publishTime_str)
            current_time = datetime.datetime.now()
            time_difference = current_time - publishTime
            if time_difference.days <= days:  # Checking if the time difference is within 3 days
                # print("The publishTime is within the past few days.",publishTime)
                # name=data['rows'][i]['id']
                url='https://supplier.dfzq.com.cn/announcementView?id='+result['id']
                urls[result['title']] = url
                # urls[data['rows'][i]['noticeTitle'] ] = url
                # urls.append(url)
                # print(url)
            # else:
            #     print("The publishTime is not within the past few days.")
        return urls
    else:
        print("请求失败，状态码：" + str(response.status_code))
        return {}
 
#获取海通官网的招标文件
def get_haitong_urls(days):
    # 发送请求
    url = 'https://itms.haitong.com/api/index/notice'
    params = {
        'currentPage': '1',
        'pageSize': '30',
        'title': '',
        'noticeFrom': '',
        'noticeType': '01'
    }

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
        'Accept': 'application/json, text/plain, */*',
        'Referer': 'https://itms.haitong.com/',
        'Connection': 'close',
        'sec-ch-ua': '"Not/A)Brand";v="8", "Chromium";v="126", "Google Chrome";v="126"',
        'sec-ch-ua-mobile': '?0',
        'Authorization': 'null',
        'sec-ch-ua-platform': '"Windows"',
        'Sec-Fetch-Site': 'same-origin',
        'Sec-Fetch-Mode': 'cors',
        'Sec-Fetch-Dest': 'empty',
        'Accept-Encoding': 'gzip, deflate',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        # 其他头部信息...
    }

    response = requests.get(url, params=params, headers=headers)
    # print(response.text)
    if response.status_code == 200:
        data = response.json()
        urls = {}

        for i in range(len(data['data']['list'])):
            id = data['data']['list'][i]['id']
            publishTime_str = data['data']['list'][i]['publishTime']
            publishTime = parser.parse(publishTime_str)
            current_time = datetime.datetime.now()
            time_difference = current_time - publishTime

            if time_difference.days <= days:  # Checking if the time difference is within 3 days
                # print("The publishTime is within the past few days.",publishTime)
                # print(data['data']['data'][i]['id'])
                url='https://itms.haitong.com/#/procurnotice?type=&id='+str(data['data']['list'][i]['id'] )
                # print( url)
                urls[data['data']['list'][i]['title'] ] = url
        return urls
    else:
        print("请求失败，状态码：" + str(response.status_code))
        return {}

#获取国泰官网的招标文件
def get_guangwang_urls(days):
    # 发送请求
    url = "https://www.gtjaqh.com/pc/bulletin"
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36"
    }
    requests.packages.urllib3.disable_warnings()
    response = requests.get(url, headers=headers,verify=False)

    # 检查响应状态码
    if response.status_code == 200:
        # 解析响应内容
        soup = BeautifulSoup(response.text, "html.parser")
        script_tag = soup.find("script", id="__NEXT_DATA__")
        if script_tag:
            script_content = script_tag.string
            data = json.loads(script_content)
            bulletin_data = data["props"]["pageProps"]["BULLETIN"]

            # 筛选出catalog等于"采购公告"的id和title的值
            purchase_bulletins = [
                bulletin for bulletin in bulletin_data
                if bulletin["catalog"] == "采购公告"
            ]
            urls = {}
            for bulletin in purchase_bulletins:
                bulletin_id = bulletin["id"]
                bulletin_title = bulletin["title"]
                update_time = bulletin["update_time"]
                publishTime = datetime.datetime.strptime(update_time, "%Y-%m-%d %H:%M:%S")
                current_time = datetime.datetime.now()
                time_difference = current_time - publishTime

                if time_difference.days <= days:  # Checking if the time difference is within 2 days
                    url = f'https://www.gtjaqh.com/pc/a/{bulletin_id}'
                    # print(url)
                    urls[bulletin_title] = url

            return urls
        else:
            print("Script tag not found.")
    else:
        print("Request failed with status code:", response.status_code)

    return {}

#获取国泰官网的招标文件采购平台2024/05/22 新增
def get_guangwang_caigou_urls(days):
    from datetime import datetime, timedelta
    # 发送请求
    url = 'http://www.gtja.com/content/info-open/supplier.html'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
        'sec-ch-ua': '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Windows"',
        'Upgrade-Insecure-Requests': '1',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
        'Sec-Fetch-Site': 'none',
        'Sec-Fetch-Mode': 'navigate',
        'Sec-Fetch-User': '?1',
        'Sec-Fetch-Dest': 'document',
        'Accept-Encoding': 'gzip, deflate',
        'Accept-Language': 'zh-CN,zh;q=0.9'
    }

    response = requests.get(url, headers=headers)
    urls = {}
    soup = BeautifulSoup(response.text, 'html.parser')

    # 获取当前时间和当前时间前三天的日期
    current_time = datetime.now()
    # three_days_ago = current_time - timedelta(days=3)

    h3_contents = []  # 存储符合条件的h3内容
    li_tags = soup.find_all('li')
    for li in li_tags:
        span_tag = li.find('span')
        if span_tag:
            span_text = span_tag.get_text().strip()
            try:
                span_date = datetime.strptime(span_text, "发布时间：%Y年%m月%d日")
                # print(span_date)
                if current_time - span_date <= timedelta(days):
                    h3_tag = li.find('h3')
                    if h3_tag:
                        # h3_contents.append(h3_tag.get_text().strip())
                        urls[h3_tag.get_text().strip()]=url
            except ValueError:
                pass
    # 将符合条件的h3内容用分号隔开输出
    # output = '\n'.join(h3_contents)
    # urls[output] = url
    return urls
#获取上海证券官网的招标文件采购平台2024/08/12 新增
def get_shanghaizhengquan_urls(days):

    url = 'https://www.shzq.com/bwsp/common'

    headers = {
        'Host': 'www.shzq.com',
        'Connection': 'close',
        'Content-Length': '247',
        'sec-ch-ua': '"Not)A;Brand";v="99", "Google Chrome";v="127", "Chromium";v="127"',
        'Accept': 'application/json, text/javascript, */*; q=0.01',
        'sec-ch-ua-platform': '"Windows"',
        'X-Requested-With': 'XMLHttpRequest',
        'sec-ch-ua-mobile': '?0',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36',
        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
        'Origin': 'https://www.shzq.com',
        'Sec-Fetch-Site': 'same-origin',
        'Sec-Fetch-Mode': 'cors',
        'Sec-Fetch-Dest': 'empty',
        'Referer': 'https://www.shzq.com/about/news.html?type=zbxx',
        'Accept-Encoding': 'gzip, deflate',
        'Accept-Language': 'zh-CN,zh;q=0.9'
    }

    data = {
        'req_url': '/bbizp/articles',
        'req_http': 'get',
        'web_categoryCodes': 'uda_web_zjsz_zbxx',
        'web_pageIndex': '1',
        'web_pageSize': '10',
        'web_authAppToken': '258b2d917e97b9e5',
        'web_publishDateBegin': '2010-01-01',
        'web_currentPage': 'https://www.shzq.com/about/news.html?type=zbxx'
    }

    response = requests.post(url, headers=headers, data=data)
    # print(response.text)
    # response = requests.get(url, params=params, headers=headers)
    # print(response.text)
    if response.status_code == 200:
        data = response.json()
        urls = {}

        for i in range(len(data['dataList'])):
            id = data['dataList'][i]['id']
            publishTime_str = data['dataList'][i]['publishDate']
            publishTime = parser.parse(publishTime_str)
            current_time = datetime.datetime.now()
            time_difference = current_time - publishTime

            if time_difference.days <= days:  # Checking if the time difference is within 3 days
                # print("The publishTime is within the past few days.",publishTime)
                # print(data['data']['data'][i]['id'])
                url='https://www.shzq.com/about/newsContent.html?pageId='+str(data['dataList'][i]['id'] )
                # print( url)
                urls[data['dataList'][i]['title'] ] = url
        return urls
    else:
        print("请求失败，状态码：" + str(response.status_code))
        return {}
#获取国泰etu365
def get_guotai_urls(days):
    url = "https://www.etu365.com/api/v1/ztb/listzbggscreen_new"
    params = {
        "iszb": "1",
        "zbr": "",
        "zbdljg": "",
        "time": "",
        "keyword": "国泰君安",                                                                                    
        "gglx": "0",
        "pageSize": "10",
        "page": "1"
    }
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36",
        "Referer": "https://www.etu365.com/notice?id=1&zbr=&zbdljg=&time=&keyword=%E5%9B%BD%E6%B3%B0%E5%90%9B%E5%AE%89",
        "Cookie": "HWWAFSESID=706c57305724d5cff8; HWWAFSESTIME=1698720360529",
        "token": "null"
    }

    response = requests.get(url, params=params, headers=headers)


    if response.status_code == 200:
        data = response.json()
        # print(data)
        urls = {}
        for i in range(len(data['data']['data'])):
            id = data['data']['data'][i]['id']
            # print(len(data['data']))
            publishTime_str = data['data']['data'][i]['ggrq_begin']
            publishTime = parser.parse(publishTime_str)
            current_time = datetime.datetime.now()
            time_difference = current_time - publishTime
            # print(publishTime_str)
            if time_difference.days <= days:  # Checking if the time difference is within 3 days
                # print("The publishTime is within the past few days.",publishTime)
                # print(data['data']['data'][i]['id'])
                url='https://webapp.etu365.com/ProjectMgr/ZBGG?ggid='+str(data['data']['data'][i]['id'] )
                # print( url)
                urls[data['data']['data'][i]['bdmc'] ] = url
        return urls
    else:
        print("请求失败，状态码：" + str(response.status_code))
        return {}
def get_nongyin(days):
    url = "http://www.cfcpn.com/jcw/noticeinfo/noticeInfo/dataNoticeList"

    payload = {
        "noticeType": "",
        "pageNo": "",
        "noticeState": "1",
        "isValid": "1",
        "orderBy": "publish_time desc",
        "noticeContent": "",
        "briefContent": "农银汇理",
        "noticeTitle": "",
        "purchaseName": "",
        "purchaseId": "",
        "categoryLabName": "",
        "beginPublishTime": "",
        "endPublishTime": "",
        "areaProvince": "",
        "labelAllId": ""
    }

    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36",
        "Referer": "http://www.cfcpn.com/jcw/sys/index/goUrl?url=modules/sys/login/list&amp;column=cggg",
        "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
        "X-Requested-With": "XMLHttpRequest",
        "Accept-Encoding": "gzip, deflate",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Connection": "close",
        "Cookie": "pageNo=0; pageSize=10"
    }

    response = requests.post(url, data=payload, headers=headers)

    if response.status_code == 200:
        data = response.json()
        urls = {}
        for i in range(len(data['rows'])):
            id = data['rows'][i]['id']
            publishTime_str = data['rows'][i]['publishTime']
            publishTime = datetime.datetime.strptime(publishTime_str, "%Y-%m-%d %H:%M:%S.%f")
            current_time = datetime.datetime.now()
            time_difference = current_time - publishTime

            if time_difference.days <= days:  # Checking if the time difference is within 3 days
                # print("The publishTime is within the past few days.", publishTime)
                url = 'http://www.cfcpn.com/jcw/sys/index/goUrl?url=modules/sys/login/detail&column=undefined&searchVal=' + data['rows'][i]['id']
                urls[data['rows'][i]['noticeTitle']] = url
                # print('http://www.cfcpn.com/jcw/sys/index/goUrl?url=modules/sys/login/detail&column=undefined&searchVal=' + data['rows'][i]['id'])
            # else:
            #     print("The publishTime is not within the past few days.")

        return urls
    else:
        print("请求失败，状态码：" + str(response.status_code))
        return {}
# 获取yilian网招标信息 2024/9/2    
def get_yilianwang(days):
    url = "https://onelinkplus.com/api/saas-portal/noauth/trans/trade/pageEs"
    headers = {
        "Host": "onelinkplus.com",
        "Connection": "close",
        "Content-Length": "390",
        "sec-ch-ua": "\"Chromium\";v=\"128\", \"Not;A=Brand\";v=\"24\", \"Google Chrome\";v=\"128\"",
        "Accept": "application/json, text/plain, */*",
        "sec-ch-ua-platform": "\"Windows\"",
        "sec-ch-ua-mobile": "?0",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
        "Content-Type": "application/json;charset=UTF-8",
        "Origin": "https://onelinkplus.com",
        "Sec-Fetch-Site": "same-origin",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Dest": "empty",
        "Referer": "https://onelinkplus.com/",
        "Accept-Encoding": "gzip, deflate",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Cookie": "HWWAFSESID=ed0e7738ede363f223; HWWAFSESTIME=1725257712325; TODAY_NO_MORE_REMINDER=0; noticeLength=1; sajssdk_2015_cross_new_user=1; org.springframework.web.servlet.i18n.CookieLocaleResolver.LOCALE=zh_CN; org.springframework.web.servlet.i18n.CookieLocaleResolver.LOCALE2=zh_CN; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%22undefined%22%2C%22first_id%22%3A%22191b16c3c15581-0d8992149c7a0d-26001151-1764000-191b16c3c16888%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22%24device_id%22%3A%22191b16c3c15581-0d8992149c7a0d-26001151-1764000-191b16c3c16888%22%7D; COMPANY_COMPLETE_STATUS_undefined=0; checkIsWanbangGroup=1; checkIsFosunGroup=1; isExperience=; companyName=undefined; regeisterType=; isCompanyManager=; isRemember=100; rememberInfo=BImp5GES3WFc9q6gw7YWMcJVVISmDL4SrC26vcd/M5yQMDCIAQxJGWPWTYJgzohnoLvs/DFs8e3z8nZN3UlCXRxaTZdIoLchqaKci76OoLI="
    }

    data = {
        "dataSource": "",
        "pageNum": 1,
        "pageSize": 20,
        "tradePattern": "882296043327582208",
        "noticeType": "1,2",
        "businessName": "",
        "tendererName": "",
        "bidderName": "",
        "releaseEndTime": "",
        "releaseStartTime": None,
        "purchaseProjectType": "",
        "purchaseMode": "",
        "platformId": "882289088363483137",
        "industry": "",
        "budgetRange": "",
        "categoryName": "",
        "tendererChooseNames": "",
        "classifySize": 0,
        "fieldName": "",
        "keySort": []
    }

    response = requests.post(url, json=data, headers=headers)
    if response.status_code == 200:
        data = response.json()
        # print(data)
        urls = {}
        for i in range(len(data['data']['list'])):
            id = data['data']['list'][i]['id']
            publishTime_timestamp = data['data']['list'][i]['releaseTime'] / 1000  # 将时间戳转换为秒
            publishTime = datetime.datetime.fromtimestamp(publishTime_timestamp, tz=pytz.UTC)  # 将时间戳转换为日期时间对象
            current_time = datetime.datetime.now(pytz.UTC)  # 获取当前时间
            time_difference = current_time - publishTime
            ##获取服务品类关键字
            guanjianzi = 'IT'
            values= str(data['data']['list'][i]['categoryName'] )
            if time_difference.days <= days and guanjianzi in values :  
                url='https://onelinkplus.com/#/trade-info-detail?id='+str(data['data']['list'][i]['id'] )+"&noticeType="+str(data['data']['list'][i]['noticeType'] )+"&publishStatus="+str(data['data']['list'][i]['publishStatus'] )
                print( url)
                # urls[data['data']['list'][i]['tendererName'] ] = url
                urls[data['data']['list'][i]['businessName'] ] = url 
        return urls
    else:
        print("请求失败，状态码：" + str(response.status_code))
        return {}   
#####################################################################
def send_email(urls,days):
    # 邮箱服务器的地址和端口
    smtp_server = 'smtp.qiye.aliyun.com'
    smtp_port = 465

    # 发件人邮箱账号和密码
    sender_email = ''
    sender_password = ''

    # 收件人邮箱账号
    receiver_email = ''
    # receiver_email = ''
    # 构造邮件内容
    message = MIMEMultipart("alternative")
    message['From'] = '招标大师'
    # 获取当前日期和时间
    current_datetime = datetime.datetime.now()

    # 计算前三天的日期
    previous_datetime = current_datetime - datetime.timedelta(days)

    # 格式化日期和时间
    formatted_current_datetime = current_datetime.strftime('%H:%M:%S, %Y-%m-%d')
    formatted_previous_datetime = previous_datetime.strftime('%H:%M:%S, %Y-%m-%d')

    # 构建邮件主题
    subject = f'【Urgent】招标文件提醒，别让机会溜走！（{formatted_previous_datetime}至{formatted_current_datetime}）'

    # 设置邮件主题
    message['Subject'] = subject

    # 定义HTML模板
    html_template = """
    <html>
<head>
<style>
    body {
        font-family: 'Comic Sans MS', cursive, sans-serif;
        font-size: 16px;
        color: #333;
    }

    h1 {
        font-family: 'Arial', sans-serif;
        font-size: 24px;
        color: #ff00ff;
        text-decoration: underline;
    }

    p {
        margin-bottom: 10px;
    }

    a {
        color: #0066cc;
        text-decoration: none;
        font-weight: bold;
    }
    </style>
    </head>
    <body>
       
        <p>亲爱的小伙伴们，</p>
        <p>我是你们的机会捕捉者，特地为你们带来了一些好消息！</p>
        <p>请查收以下信息：</p>
        {% for key, value in urls.items() %}
        <p>请点击<a href='{{ value }}'>{{ key }}</a>查看更多内容。</p>
        {% endfor %}
        <p>祝你们好运！</p>
        <p>机会捕捉者 敬上</p>
    </body>
    </html>
    """

    # 创建Jinja2模板对象
    template = Template(html_template)
    # 渲染模板并传递参数
    if len(urls) == 0:
        print("最近没有招标文件哦，请耐心等待！")
        return
    else:
        html = template.render(urls=urls)
        # 将HTML内容转换为MIMEText对象
        part = MIMEText(html, "html")
        # 将MIMEText对象添加到MIMEMultipart对象中
        message.attach(part)
        try:
            # 连接到邮件服务器
            server = smtplib.SMTP_SSL(smtp_server, smtp_port)
            server.login(sender_email, sender_password)
            receivers = receiver_email.split(',')
            server.sendmail(sender_email, receivers, message.as_string())
            server.quit()
            print("邮件发送成功")
        except Exception as e:
            print("邮件发送失败:", str(e))

def main():
    # urls = get_haitong_urlss(days)
    days = 3
    urls = {**get_guangwang_urls(days), **get_guotai_urls(days), **get_nongyin(days), **get_dongfang(days),**get_guangwang_caigou_urls(days),**get_haitong_urls(days),**get_shanghaizhengquan_urls(days),**get_yilianwang(days)}
    # urls = { **get_yilianwang(days)}
    send_email(urls,days)

if __name__ == "__main__":
    main()

招投标行业114个数据源盘点
网页爬虫与数据采集 · 八爪鱼
03-25 2867
招投标是政府单位与企业之间一种重要的采购手段，随着大数据时代的到来，招投标行业迎来了高速发展。据统计，目前我国依法应招项目的招标率已达90％，招投标领域正在不断发展扩大。 招投标网站的建立，不仅能汇集招标单位的全部信息和数据，帮助各地的投标企业集中获取招标信息，使得招投标再也不用受空间、地域的限制，帮助企业进行行业调研，获取更多销售线索以及对政府政策进行研究分析；同时招投标网站也有助于采购单位选择更为优秀的合作企业；公开化的模式有利于政府对招投标过程进行监督管理。 目前招投标网站主要分为三大类，包括：
编写python爬虫 获取中华英才网全网工资数据
Qton_CSDN的博客
05-07 4568
做数据分析 数据挖掘，第一步是获取数据，在这里，我们要分析现今全国各地各个职业的工资情况。 我们选择较为权威的‘中华英才网’，编写python爬虫获取该网站上的各个招聘信息说给出的工资，再取其行业工资的平均值，即为该行业目前的大概工资。 1 以深圳为例 如‘数据挖掘’这个职业在深圳查找的情况如上图。代码里我们去掉‘面议‘，工资3000-6000这样的我们取其中值4500写ex
「从零开始学爬虫」以采集中国国际招标网为例
Liguangmei的博客
05-30 2207
提取招标标题，时间，公告类型，每一条招标信息的详细页面的url以及省份
投标网源代码 通过JAVA爬网上所有的政府招标网数据
05-18
投标网源代码 通过JAVA爬网上所有的政府招标网数据，前端展示！
基于python爬虫大数据技术的招投标信息检索系统
QQ242219979的博客
05-12 1403
第二，同体监督难护市场公正，在目前的投资体制和管理体制条件下，各有关部门对投标活动实施人，没有形成制衡机制，更难于实施有效监督；第五，评标环节“漏洞百出”，评标是招标环节中最关键的，而在我国，由于种种原因，导致评标不够科学合理；第六，招标人规避“公招”花样多，据报告，主要有4种方式，一是把项目化整为零，二是把应当公开招标的项目千方百计找各种理由将变为邀请招标，三是在信息发布上做文章，四是一些业主单位为了私利，不肯把自己应当招投标的项目进行公开招投标，而是交给自己所属的单位完成。
python爬虫逆向|某招投标信息网站返回结果解密
weixin_43459158的博客
05-31 1129
网站链接 aHR0cDovL2N0YnBzcC5jb20vIy8= (base64解密后可见) 正文 网站搜索内容是后台加载的，且响应的内容是加密后的 然后大概看一下堆栈，除了vue,bs等资源的js，这个文件还是蛮可疑的 进去后，也没多少行代码，而且，看头部信息，加密的文件引用和加密方式一目了然 既然是des，那肯定还是有公钥的，在这个utils.js寻找下看有没有公钥，果然一目了然，甚至连注释都有，挺良心的啊，嗯 这就简单多了，直接吧原文js复制到本地，然后npm install crypto-
爬虫-金采网数据(2018-11-19)
当法律与事业相遇
11-19 1696
文章目录爬虫地址环境爬虫代码致谢 爬虫地址 http://www.cfcpn.com/plist/caigou?pageNo=1&amp;amp;kflag=0&amp;amp;keyword=&amp;amp;keywordType=&amp;amp;province=&amp;amp;city=&amp;amp;typeOne=&amp;amp;ptpTwo=,, 环境 python3.6.5 爬虫代码 # -*- coding:utf-8*
爬虫实践：陕西招投标爬虫（三个网站）xpath封装并exe
weixin_48572116的博客
08-02 2657
最近做的一个陕西省政府招投标爬虫的封装的代码项目，并exe
【行业了解】天眼查、企查查、启信宝、爱企查
热门推荐
wsdchong的博客
11-24 2万+
【行业了解】天眼查、企查查、启信宝、爱企查 前言 今天写《【行业分析】矿泉水行业》的时候，简单从两个三方平台获取信息，不过毕竟是第三方平台，我不了解的话我心里不放心，于是简单了解一下查企业的第三方平台。 11月22写《【企业分析】索菲亚》的时候用了三个平台查数据，于是先从这三家展开了解。 《爱企查》、《天眼查》、《企查查》、《启信宝》 目前我简单从爱企查以及萝卜投研，这两个渠道来获取信息。虽然借助了这两个第三方平台，但是这些第三方平台是整合了不错的一手数据的平台，我自己去搜集一手数据会花费大量时间。 文
各行业领域数据集整理送给大家！
简书博客搬家测试账号
03-12 3224
欢迎关注天善智能，我们是专注于商业智能BI，人工智能AI，大数据分析与挖掘领域的垂直社区，学习，问答、求职一站式搞定！ 对商业智能BI、大数据分析挖掘、机器学习，python，R等数据领域感兴趣的同学加微信：tstoutiao，邀请你进入数据爱好者交流群，数据爱好者们都在这儿。 作者：爱德宝器 Python爱好者社区专栏作者公众号：数据管道摘要"...
抓取中国招标网数据源码
02-01
自动抓取中国招标网数据，并保存到本地数据库。C#源码，找招标信息再也不用去中国招标网一页一页的翻了。提供源码，如果有其他的公共资源想本地化可以同样的办法处理。此代码仅供学习，不可用于非法用途。
互联网金融爬虫怎么写－第二课 雪球网股票爬虫（正则表达式入门）
05-23
系列教程《互联网金融爬虫怎么写》第一课，详细讲解了如何开发爬虫爬取一些有名的电商网站数据。对于学习爬虫的开发者有很大帮助，内含大量可直接运行的源码。 第五课主要以爬取股票网站上的信息为例，主要讲解了爬虫里经常使用的正则表达式。
知识图谱类产品-开题报告构想（一）
weixin_51117061的博客
11-26 2613
知识图谱类产品-开题报告构想 整体技术路线是GNN，降低前期语料的爬取成本，化团队力量为自己用，开始日益发现团队力量的重要性，在一个团队中可以相互成就一个产品，形成能力的互补和简历的递进，but像去年那种技术组的过于分裂，导致技术壁垒在团队内部产生的现象一定要注意避免，做到技术的可互通性的同时，也要做好项目的完善工作。 1.信息源获取难度以及合法性 自然语言处理较为依赖于信息源的获取，本团队不希望将重心放在前期的语料爬取中，虽然这对于结果很重要，但是发展初期的主要方向还是把整个框架能够成功跑出来...
如何利用Python高效抓取招投标信息？一文解锁自动化采集秘籍
最新发布
zhou6343178的博客
07-08 2240
本文将深入探讨如何运用Python技术构建一个高效、稳定的招投标信息采集系统，旨在帮助企业和开发者掌握自动化数据抓取的核心技巧。通过实战示例与策略解析，揭秘从零开始搭建招投标信息爬虫的全过程，提升数据获取效率，助力企业决策。
招标网站信息爬取
u010643777的专栏
11-30 4174
招标网站爬取，采用python语言实现的爬虫
maven初体验
hiwzg
04-18 306
       之所以看Maven，是因为看到一些文章，讲Maven是如何支持项目构建的，是下一代Ant……。于是乎，经不起诱惑，也开始接触了一下Maven。        Maven的基本概念先不讲，这里主要记录我的一些体会。        我对Maven印象最为深刻的是：Maven其实不是Ant，Ant通过Task完成很多关于项目编译，打包之类的问题，然后提供了很多的Task，完成其它的工作...
Python爬虫 selenium抓取企查查招投标
whn18723613485的博客
01-03 1342
企查查最新中标企业抓取
爬虫入库
qq_32394351的博客
12-20 4350
爬某免费招标网站，不涉及版权。直接上代码： import requests import json # from pprint import pprint import time # from lxml import etree import re import sqlite3 import datetime class sql_con: def __init__(self, sql...
招投标信息采集与查询
u012057217的博客
03-16 1209
这次的案例主要是演示实时采集多个网站招标信息，发现最新招标信息立即提醒报警提醒并保存在软件的历史记录中，长期数据累积形成庞大的招标信息历史记录中，以后可随时根据日期和关键词查询。下图为这次案例的监控对象的其中一个网站： 随后就点击添加，把网站复制到对应的地方，并点击自动获取编码。（按情况设置监控周期，这里设置监控周期为90秒。） 这里添...
Python3.6实现网站目录全面遍历爬虫
网站文件夹目录遍历是网络爬虫技术中的一个重要应用，它允许爬虫程序遍历网站的所有文件和文件夹，以获取网站的结构信息、文件列表或特定资源。通过编写程序来实现目录遍历，可以自动化地完成这一过程，提高效率并...
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值