主题:论文代码统计
所有论文出现代码的相关统计;使⽤正则表达式统计代码连接、⻚数和图表数据;
import seaborn as sns
from bs4 import BeautifulSoup
import re
import json
import pandas as pd
import matplotlib.pyplot as plt
数据处理
在原始arxiv数据集中作者经常会在论文的comments
或abstract
字段中给出具体的代码链接,所以我们需要从这些字段里面找出代码的链接。
- 确定数据出现的位置;
- 使用正则表达式完成匹配;
- 完成相关的统计;
def readArxivFile(path, columns=['id', 'submitter', 'authors', 'title', 'comments', 'journal-ref', 'doi',
'report-no', 'categories', 'license', 'abstract', 'versions',
'update_date', 'authors_parsed'], count=None):
'''
定义读取文件的函数
path: 文件路径
columns: 需要选择的列
count: 读取行数
'''
data = []
with open(path, 'r') as f:
for idx, line in enumerate(f):
if idx == count:
break
d = json.loads(line)
d = {col : d[col] for col in columns}
data.append(d)
data = pd.DataFrame(data)
return data
data1 = readArxivFile('arxiv-metadata-oai-snapshot.json', ['id', 'abstract', 'categories', 'comments'])
data1.head(2)
id | abstract | categories | comments | |
---|---|---|---|---|
0 | 0704.0001 | A fully differential calculation in perturba... | hep-ph | 37 pages, 15 figures; published version |
1 | 0704.0002 | We describe a new algorithm, the $(k,\ell)$-... | math.CO cs.CG | To appear in Graphs and Combinatorics |
#下面会对data进行重复赋值,建议将data复制出来,不用重复读数据
data = data1.copy()
#正则表达式匹配,XX pages ,这里可以将原来的[1-9][0-9]替换为\d+
data['pages'] = data['comments'].apply(lambda x : re.findall('\d+ pages',str(x)))
data.head(2)
id | abstract | categories | comments | pages | |
---|---|---|---|---|---|
0 | 0704.0001 | A fully differential calculation in perturba... | hep-ph | 37 pages, 15 figures; published version | [37 pages] |
1 | 0704.0002 | We describe a new algorithm, the $(k,\ell)$-... | math.CO cs.CG | To appear in Graphs and Combinatorics | [] |
#筛选出有pages的论文
data = data[data['pages'].apply(len)>0]
# 由于匹配得到的是一个list,如['19 pages'],需要进行转换
data['pages'] = data['pages'].apply(lambda x: int(x[0].replace(' pages','')))
data.head(2)
id | abstract | categories | comments | pages | |
---|---|---|---|---|---|
0 | 0704.0001 | A fully differential calculation in perturba... | hep-ph | 37 pages, 15 figures; published version | 37 |
2 | 0704.0003 | The evolution of Earth-Moon system is descri... | physics.gen-ph | 23 pages, 3 figures | 23 |
对pages进行统计
data['pages'].describe().astype(int)
count 1089208
mean 17
std 22
min 0
25% 8
50% 13
75% 22
max 11232
Name: pages, dtype: int32
接下来按照分类统计论文页数,选取了论文的第一个类别的主要类别:
# 选择主要类别
#拆分出空格
data['categories'] = data['categories'].apply(lambda x: x.split(' ')[0])
#拆分出"."
data['categories'] = data['categories'].apply(lambda x: x.split('.')[0])
data.head(2)
id | abstract | categories | comments | pages | |
---|---|---|---|---|---|
0 | 0704.0001 | A fully differential calculation in perturba... | hep-ph | 37 pages, 15 figures; published version | 37 |
2 | 0704.0003 | The evolution of Earth-Moon system is descri... | physics | 23 pages, 3 figures | 23 |
数据可视化
# 每类论文的平均页数
plt.figure(figsize=(12, 6))
data.groupby(['categories'])['pages'].mean().plot(kind='bar')
<AxesSubplot:xlabel='categories'>
data.head(2)
id | abstract | categories | comments | pages | |
---|---|---|---|---|---|
0 | 0704.0001 | A fully differential calculation in perturba... | hep-ph | 37 pages, 15 figures; published version | 37 |
2 | 0704.0003 | The evolution of Earth-Moon system is descri... | physics | 23 pages, 3 figures | 23 |
对论文图表个数进行抽取
data['figures'] = data['comments'].apply(lambda x: re.findall('[1-9][0-9]* figures', str(x)))
data = data[data['figures'].apply(len) > 0]
data['figures'] = data['figures'].apply(lambda x: int(x[0].replace(' figures', '')))
data.head(2)
id | abstract | categories | comments | pages | figures | |
---|---|---|---|---|---|---|
0 | 0704.0001 | A fully differential calculation in perturba... | hep-ph | 37 pages, 15 figures; published version | 37 | 15 |
2 | 0704.0003 | The evolution of Earth-Moon system is descri... | physics | 23 pages, 3 figures | 23 | 3 |
对论文的代码链接进行提取
# 筛选包含github的论文
data_with_code = data[
(data.comments.str.contains('github')==True)|
(data.abstract.str.contains('github')==True)
]
data_with_code.loc[:,'text'] = data_with_code.loc[:,'abstract'].fillna('') + data_with_code.loc[:,'comments'].fillna('')
# 使用正则表达式匹配论文
pattern = '[a-zA-z]+://github[^\s]*'
data_with_code.loc[:,'code_flag'] = data_with_code.loc[:,'text'].str.findall(pattern).apply(lambda x : 0 if len(x) <1 else 1)
data_with_code.head(2)
id | abstract | categories | comments | pages | figures | text | code_flag | |
---|---|---|---|---|---|---|---|---|
253172 | 1103.5904 | Solar tomography has progressed rapidly in r... | astro-ph | 21 pages, 6 figures, 5 tables | 21 | 6 | Solar tomography has progressed rapidly in r... | 0 |
254226 | 1104.0672 | We describe a hybrid Fourier/direct space co... | astro-ph | 10 pages, 6 figures. Submitted to Astronomy an... | 10 | 6 | We describe a hybrid Fourier/direct space co... | 1 |
对论文按照类别进行绘图
data_with_code = data_with_code[data_with_code['code_flag'] == 1]
plt.figure(figsize=(12, 6))
data_with_code.groupby(['categories'])['code_flag'].count().plot(kind='bar')
<AxesSubplot:xlabel='categories'>