爬取知网《钢结构》期刊所有文章的摘要
import requests
import re
headers={'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.9 Safari/537.36'}
xl=['01','02','03','04','05','06','07','08','09','10','11','12']
for i in range(2010,2020):
for j in range(len(xl)):
baseurl='http://navi.cnki.net/knavi/JournalDetail/GetArticleList?year={}&issue={}&pykm=GJIG&pageIdx=0&pcode=CJFD'.format(i,xl[j])
print(baseurl)
basedata=requests.get(baseurl,headers=headers)
basedata=basedata.text
filename=re.findall(r'amp;filename=(.*?)&tableName',basedata)
tablename=re.findall(r'tableName=(.*?)&url=',basedata)
for m in range(len(filename)):
detailurl='http://kns.cnki.net/kcms/detail/detail.aspx?dbcode=CJFD&filename={}&dbname={}'.format(filename[m],tablename[m])
detaildata=requests.get(detailurl,headers=headers)
detaildata=detaildata.text
title=re.findall(r'<h2 class="title">(.*?)</h2><a class=',detaildata)
zhaiyao=re.findall(r'</label><span id="ChDivSummary" name="ChDivSummary">(.*?)</span><span>',detaildata)
print(title[0])
try:
with open(f'abstract.txt','a+') as f:
f.write('《')
f.write(title[0])
f.write('》')
f.write('——')
f.write(zhaiyao[0])
f.write('\n')
except Exception as e:
print(e)