from bs4 import BeautifulSoup
from MysqlTest import *
import requests
import time
import datetime
#是否打印相关信息
isprint = 0
def caiji_byid(cid, url, soupselect, dateselect, titleselect, urlselect, urlbase ,datetype = '%Y-%m-%d'):
"""通过id采集相应数据并存入数据库"""
t0 = time.perf_counter()
n = 0
for sp in caiji_byurl(url, soupselect):
cdate = sp.select(dateselect)[0].text.strip().strip()
cdate = str(datetime.datetime.strptime(cdate, datetype))
ctitle = sp.select(titleselect)[0].text.strip()
url = urlbase + sp.select(urlselect)[0].attrs['href'].strip()
if(isprint == 1):
print(cid, cdate, ctitle, url)
n += caiji_save(cid, cdate, ctitle, url)
print_results(cid, n, t0)
def caiji_byid_imf(cid, url, soupselect, urlbase):
"""通过id采集相应数据并存入数据库,节点为平行结构"""
t0 = time.perf_counter()
n = 0
for sp in caiji_byurl(url, soupselect):
cdate = sp.parent.next_sibling.select("span")[0].text.strip()
cdate = str(datetime.datetime.strptime(cdate, '%B %d , %Y'))
ctitle = sp.text.strip()
url = urlbase + sp.attrs['href']
if (isprint == 1):
print(cid, cdate, ctitle, url)
n += caiji_save(cid, cdate, ctitle, url)
print_results(cid, n, t0)
def caiji_save(cid, cdate, ctitle, url):
sql = "select count(*) from caiji where cid=%d and cdate=%s and ctitle=%s " % (
cid, repr(cdate), repr(ctitle))
if (int(querysql(sql)[0][0]) == 0):
sql = 'insert into caiji(cid,cdate,ctitle,url) values(%d,%s,%s,%s)' % (
cid, repr(cdate), repr(ctitle), repr(url))
cmdsql(sql)
print(str(cid), cdate, ctitle, url)
return 1
else:
return 0
def caiji_byurl(url, soupselect):
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36'
}
html = requests.get(url, headers=headers).content
soup = BeautifulSoup(html, "lxml").select(soupselect)
return soup
def caiji_zhongguooumengshanghui():
"""采集中国欧盟商会信息"""
""""""
cid = 1
url = "https://www.europeanchamber.com.cn/en/press-releases"
dateselect = ".chapter-category"
titleselect = "h3 a"
urlselect = "h3 a"
urlbase = "https://www.europeanchamber.com.cn"
soupselect = ".panel-default"
caiji_datelen(cid=cid, url=url, soupselect=soupselect, dateselect=dateselect, titleselect=titleselect,
urlselect=urlselect, urlbase=urlbase)
def caiji_datelen(cid, url, soupselect, dateselect, titleselect, urlselect, urlbase ,datetype = '%Y-%m-%d', datelen=10):
"""通过id采集相应数据并存入数据库"""
t0 = time.perf_counter()
n = 0
for sp in caiji_byurl(url, soupselect):
cdate = sp.select(dateselect)[0].text.strip()[:datelen].strip()
cdate = str(datetime.datetime.strptime(cdate, datetype))
ctitle = sp.select(titleselect)[0].text.strip()
url = urlbase + sp.select(urlselect)[0].attrs['href'].strip()
if (isprint == 1):
print(cid, cdate, ctitle, url)
n += caiji_save(cid, cdate, ctitle, url)
print_results(cid, n, t0)
def caiji_imfnews():
"""采集IFM最新信息"""
"""日期为平行结构"""
cid = 2
url = "https://www.imf.org/external/what/whatsnewenglish/what.aspx"
soupselect = "#content h4 a"
urlbase = "https://www.imf.org"
caiji_byid_imf(cid, url, soupselect, urlbase)
def caiji_unctad():
"""采集联合国贸发会议新闻"""
cid = 3
url = "https://unctad.org/en/Pages/Home.aspx"
soupselect = "#container1 .row"
urlbase = "https://unctad.org"
dateselect = "div p b"
titleselect = "a span"
urlselect = "div div a"
datetype = "%d %B %Y"
caiji_byid(cid, url, soupselect, dateselect, titleselect, urlselect, urlbase, datetype)
def caiji_OECD_pre():
"""采集OECD PRE新闻"""
cid = 4
url = "https://www.oecd-ilibrary.org/search?value51=%27igo%2Foecd%27&sortDescending=false&value5=30191110114407&operator51=AND&value1=subtype%2Fissue+OR+subtype%2Fbook&value4=20191110114407&option5=sortDate_to&value3=status%2F50embargoDate&publisherId=%2Fcontent%2Figo%2Foecd&facetNames=pub_igoId_facet+pub_themeId_facet&option3=pub_contentStatus&sortField=prism_publicationDate&option4=sortDate_from&option1=dcterms_type&facetOptions=51+52&option51=pub_igoId_facet&operator52=AND&option52=pub_themeId_facet&value52=%27theme%2Foecd-79%27"
soupselect = ".title_box"
urlbase = "https://www.oecd-ilibrary.org"
dateselect = ".search-metaitem + .comma_separated li"
titleselect = ".search_title"
urlselect = ".search_title a"
datetype = "%d %b %Y"
caiji_byid(cid, url, soupselect, dateselect, titleselect, urlselect, urlbase, datetype)
def print_results(cid, n, t0):
"""打印采集数据结果和所用时间"""
t2 = time.perf_counter()
sql = "select ctitle from leibie where cid = %d" % (cid)
leibie = querysql(sql)[0][0]
print("采集ID", cid, leibie, "共", n, "条记录", "耗时:", t2 - t0)
#1采集中国欧盟商会新闻
caiji_zhongguooumengshanghui()
#2采集IMF新闻
caiji_imfnews()
#3采集联合国贸发会议新闻
caiji_unctad()
#4采集OECD新闻
caiji_OECD_pre()