Day03-数据分析实战-论文代码统计(DataWhale)

主题：论文代码统计

所有论文出现代码的相关统计；使⽤正则表达式统计代码连接、⻚数和图表数据；

import seaborn as sns
from bs4 import BeautifulSoup
import re 
import json 
import pandas as pd 
import matplotlib.pyplot as plt

数据处理

在原始arxiv数据集中作者经常会在论文的comments或abstract字段中给出具体的代码链接，所以我们需要从这些字段里面找出代码的链接。

• 确定数据出现的位置；
• 使用正则表达式完成匹配；
• 完成相关的统计；

def readArxivFile(path, columns=['id', 'submitter', 'authors', 'title', 'comments', 'journal-ref', 'doi',
       'report-no', 'categories', 'license', 'abstract', 'versions',
       'update_date', 'authors_parsed'], count=None):
    '''
    定义读取文件的函数
        path: 文件路径
        columns: 需要选择的列
        count: 读取行数
    '''
    
    data  = []
    with open(path, 'r') as f: 
        for idx, line in enumerate(f): 
            if idx == count:
                break
                
            d = json.loads(line)
            d = {col : d[col] for col in columns}
            data.append(d)

    data = pd.DataFrame(data)
    return data

data1 = readArxivFile('arxiv-metadata-oai-snapshot.json', ['id', 'abstract', 'categories', 'comments'])

data1.head(2)

	id	abstract	categories	comments
0	0704.0001	A fully differential calculation in perturba...	hep-ph	37 pages, 15 figures; published version
1	0704.0002	We describe a new algorithm, the $(k,\ell)$-...	math.CO cs.CG	To appear in Graphs and Combinatorics

#下面会对data进行重复赋值，建议将data复制出来，不用重复读数据
data = data1.copy()
#正则表达式匹配，XX pages ,这里可以将原来的[1-9][0-9]替换为\d+
data['pages'] = data['comments'].apply(lambda x : re.findall('\d+ pages',str(x)))
data.head(2)

	id	abstract	categories	comments	pages
0	0704.0001	A fully differential calculation in perturba...	hep-ph	37 pages, 15 figures; published version	[37 pages]
1	0704.0002	We describe a new algorithm, the $(k,\ell)$-...	math.CO cs.CG	To appear in Graphs and Combinatorics	[]

#筛选出有pages的论文
data = data[data['pages'].apply(len)>0]

# 由于匹配得到的是一个list，如['19 pages']，需要进行转换
data['pages'] = data['pages'].apply(lambda x: int(x[0].replace(' pages','')))

data.head(2)

	id	abstract	categories	comments	pages
0	0704.0001	A fully differential calculation in perturba...	hep-ph	37 pages, 15 figures; published version	37
2	0704.0003	The evolution of Earth-Moon system is descri...	physics.gen-ph	23 pages, 3 figures	23

对pages进行统计

data['pages'].describe().astype(int)

count    1089208
mean          17
std           22
min            0
25%            8
50%           13
75%           22
max        11232
Name: pages, dtype: int32

接下来按照分类统计论文页数，选取了论文的第一个类别的主要类别：

# 选择主要类别
#拆分出空格
data['categories'] = data['categories'].apply(lambda x: x.split(' ')[0])
#拆分出"."
data['categories'] = data['categories'].apply(lambda x: x.split('.')[0])

data.head(2)

	id	abstract	categories	comments	pages
0	0704.0001	A fully differential calculation in perturba...	hep-ph	37 pages, 15 figures; published version	37
2	0704.0003	The evolution of Earth-Moon system is descri...	physics	23 pages, 3 figures	23

数据可视化

# 每类论文的平均页数
plt.figure(figsize=(12, 6))
data.groupby(['categories'])['pages'].mean().plot(kind='bar')

<AxesSubplot:xlabel='categories'>

data.head(2)

	id	abstract	categories	comments	pages
0	0704.0001	A fully differential calculation in perturba...	hep-ph	37 pages, 15 figures; published version	37
2	0704.0003	The evolution of Earth-Moon system is descri...	physics	23 pages, 3 figures	23

对论文图表个数进行抽取

data['figures'] = data['comments'].apply(lambda x: re.findall('[1-9][0-9]* figures', str(x)))
data = data[data['figures'].apply(len) > 0]
data['figures'] = data['figures'].apply(lambda x: int(x[0].replace(' figures', '')))

data.head(2)

	id	abstract	categories	comments	pages	figures
0	0704.0001	A fully differential calculation in perturba...	hep-ph	37 pages, 15 figures; published version	37	15
2	0704.0003	The evolution of Earth-Moon system is descri...	physics	23 pages, 3 figures	23	3

对论文的代码链接进行提取

# 筛选包含github的论文
data_with_code = data[
    (data.comments.str.contains('github')==True)|
                      (data.abstract.str.contains('github')==True)
]

data_with_code.loc[:,'text'] = data_with_code.loc[:,'abstract'].fillna('') + data_with_code.loc[:,'comments'].fillna('')
# 使用正则表达式匹配论文
pattern = '[a-zA-z]+://github[^\s]*'
data_with_code.loc[:,'code_flag'] = data_with_code.loc[:,'text'].str.findall(pattern).apply(lambda x : 0 if len(x) <1 else 1)
data_with_code.head(2)

	id	abstract	categories	comments	pages	figures	text	code_flag
253172	1103.5904	Solar tomography has progressed rapidly in r...	astro-ph	21 pages, 6 figures, 5 tables	21	6	Solar tomography has progressed rapidly in r...	0
254226	1104.0672	We describe a hybrid Fourier/direct space co...	astro-ph	10 pages, 6 figures. Submitted to Astronomy an...	10	6	We describe a hybrid Fourier/direct space co...	1

对论文按照类别进行绘图

data_with_code = data_with_code[data_with_code['code_flag'] == 1]
plt.figure(figsize=(12, 6))
data_with_code.groupby(['categories'])['code_flag'].count().plot(kind='bar')

<AxesSubplot:xlabel='categories'>