大数据实验考试傻瓜模板

1.数据清洗

import pandas as pd
from sklearn.cluster import KMeans
from sklearn.preprocessing import StandardScaler
import matplotlib.pyplot as plt

# 读取原始 CSV 文件，使用 'utf-8' 编码方式
df = pd.read_csv("D:/duanhuibgk/air_data.csv", encoding='ANSI')

# 数据缺失值处理，使用均值填充
print(df)
df.fillna(df.mean(), inplace=True)

# 数据属性规约，将字符类型的数据转换为数值类型

# 数据清洗和变换，例如删除异常值
df = df[(df['AVG_INTERVAL'] > 0) & (df['LAST_TO_END'] > 0)]

# 读取两列数据并进行加和操作生成新的列 "SUM_YR"
df['SUM_YR'] = df['SUM_YR_1'] + df['SUM_YR_2']

2.相关系数

print("源数据：")
print(df)
print("EDUCATION和MARRIAGE之间的相关系数：")
df['MARRIAGE'].corr(df['EDUCATION'])
print(df['MARRIAGE'].corr(df['EDUCATION']))
print("三门成绩的相关系数：")
df.loc[:,['EDUCATION','PAY_0','MARRIAGE']].corr()
print(df.loc[:,['EDUCATION','PAY_0','MARRIAGE']].corr())

3. 分布分析

import pandas
from pandas import read_csv
df = read_csv(r"/Users/y/Desktop/UCI_Credit_Card.csv",sep = ',',encoding='utf8')
print("源数据：")
print(df)
bins = [min(df.AGE)-1,30,40,max(df.AGE)+1]
print(bins)
labels = ['30岁以下','30岁到40分','40岁以上']
print(labels)

print("年龄分层：")
hierarchy = pandas.cut(df.AGE,bins,labels = labels)
print(hierarchy)
print("添加年龄分层：")
df['AGE hierarchy'] = hierarchy
print(df)

df1 = df.groupby(by = ['AGE hierarchy'])['AGE'].agg([("number","size")])
print(df1)

print(labels)

4. 画点图和线图

import matplotlib.pyplot as plt
# 创建绘图对象figure,设置画布大小figsize,分辨率dpi，背景颜色facecolor可以用十六进制颜色代码
fig=plt.figure(figsize=(6,6),dpi=100,facecolor='#dfd7d7')
#划分子图ax
ax=fig.add_subplot(1,1,1)
import numpy as np
x = np.linspace(-3,3,100)
y = np.random.randn(100)
#设置中文显示
plt.rcParams['font.sans-serif'] = 'SimHei' 
plt.rcParams['axes.unicode_minus'] = False
#生成散点图
plt.grid(linestyle = ':',color = 'r',alpha = 0.5)
plt.scatter(x,y,c='k',alpha = 0.3,marker = '*',label = 'satter',lw = 1,edgecolor = 'r')
#使用marker改变点形状
plt.xlim(0,4)
plt.ylim(-3,3)
#表格设计
plt.xlabel('x轴')
plt.ylabel('y轴')
plt.axhline(y = 0,color = 'r',ls = '-.',alpha = 0.8)
plt.axvline(x = 2.0,color = 'r',ls = '-.',alpha = 0.8)
plt.legend(loc = 'upper right')
plt.title('Structure of matplotlib')
#生成sin图
x = np.linspace(1,3.5,100)
y = np.sin(x)
plt.plot(x,y,c ='y',alpha = 0.5,ls = '--',label = 'sin(x)',lw = 2)
plt.legend()
#设置平行于x/y轴的参考区域
plt.axvspan(xmin=1.5,xmax=2.5,color='blue',alpha=0.1)
plt.axhspan(ymin=-1,ymax=1,color='yellow',alpha=0.1)
plt.annotate('maxium',xy = (1.5,1),xytext = (1.3,1.3),arrowprops = {'arrowstyle':'->','connectionstyle':'arc3'})

5.直方图

import matplotlib.pyplot as plt
import numpy as np
"""
绘制直方图
data:必选参数，绘图数据
bins:直方图的长条形数目，可选项，默认为10
normed:是否将得到的直方图向量归一化，可选项，默认为0，代表不归一化，显示频数。normed=1，表示归一化，显示频率。
facecolor:长条形的颜色
edgecolor:长条形边框的颜色
"""
data = np.random.randn(10000)
plt.hist(data, bins=40, facecolor="#dfd7d7", edgecolor="k",label = 'x')# 显示横轴标签
plt.legend()
plt.xlabel("区间")# 显示纵轴标签
plt.ylabel("频数/频率")# 显示图标题
plt.title("频数/频率分布直方图")

6.等值线图

#绘制三维图像
import numpy as np
from matplotlib import cm
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
#生成数据
delta=0.2
x=np.arange(-3,3,delta)
y=np.arange(-3,3,delta)
X,Y = np.meshgrid(x,y) #生成网格点坐标矩阵
Z=X**2+Y**2
x=X.flatten() #返回一维数组
y=Y.flatten()
z=Z.flatten()
fig=plt.figure(figsize=(12,6)) #figsize设定图片宽高
#cmap指颜色，jet表示蓝-青-黄-红
ax1=fig.add_subplot(121,projection='3d')
ax1.plot_trisurf(x,y,z,cmap=cm.jet, linewidth=0.01)
plt.title("3D")
ax2=fig.add_subplot(122)
#15表示显示等高线的密集程度，数据越大画的等高线越多
cs=ax2.contour(X,Y,Z,15,cmap='jet')
ax2.clabel(cs,inline=True,fontsize=10,fmt='%1.1f')
plt.title("Contour")
plt.show()

7.三维曲面图

#绘制三维图像+各个轴的投影等高线
from matplotlib import cm
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import axes3d
fig=plt.figure(figsize=(8,6))
ax=fig.gca(projection='3d')
#生成三维测试数据
X,Y,Z=axes3d.get_test_data(0.05)
ax.plot_surface(X,Y,Z,rstride=8,cstride=8,alpha=0.3)
cset=ax.contour(X,Y,Z,zdir='z',offset=-100,cmap=cm.coolwarm)
cset=ax.contour(X,Y,Z,zdir='x',offset=-40,cmap=cm.coolwarm)
cset=ax.contour(X,Y,Z,zdir='y',offset=40,cmap=cm.coolwarm)
ax.set_xlabel('X')
ax.set_xlim(-40,40)
ax.set_ylabel('Y')
ax.set_ylim(-40,40)
ax.set_zlabel('Z')
ax.set_zlim(-100,100)
plt.show()

8.条形图

#绘制条形图
import matplotlib.pyplot as plt
import numpy as np
#生成数据
n_groups=5
#平均分和标准差
means_men=(20,35,30,35,27)
std_men=(2,3,4,1,2)
means_women=(25,32,34,20,25)
std_women=(3,5,2,3,3)
#条形图
fig,ax=plt.subplots()
#生成0,1,2,3，……
index=np.arange(n_groups)
bar_width=0.35 #条的宽度
opacity=0.4
error_config={"ecolor":"0.3"}
#条形图的第一类条
rects1=plt.bar(index,means_men,bar_width,alpha=opacity,color='b',yerr=std_men,
error_kw=error_config,label='Men')

#条形图中的第二类条
rects2=plt.bar(index+bar_width,means_women,bar_width,alpha=opacity,color='r',
yerr=std_women,error_kw=error_config,label='Women')
plt.xlabel('Group')
plt.ylabel('Scores')
plt.title('Scores by group and gender')
plt.xticks(index+bar_width,('A','B','C','D','E'))
plt.legend()
#自动调整subplot的参数给指定的填充区
plt.tight_layout()
plt.show()

9.饼图

import matplotlib.pyplot as plt
#切片将顺时针方向排列并绘制
labels='Frogs','Hogs','Dogs','Logs' #标注
sizes=[15,30,45,10] #大小
colors=['yellowgreen','gold','lightskyblue','lightcoral'] #颜色
#0.1代表第二个块从圆中分离出来
explode=(0,0.1,0,0)
#绘制
plt.pie(sizes,explode=explode,labels=labels,colors=colors,
autopct='%1.1f%%',shadow=True,startangle=90)
plt.axis('equal')
plt.show()

10.气泡图

import matplotlib.pyplot as plt
import numpy as np
x = np.random.randn(1000)
y1 = np.random.randn(len(x))
y2 = 1.8 + np.exp(x)
ax1 = plt.subplot(1,2,1)
ax1.scatter(x,y1,color='r',alpha=.3,edgecolors='white',label='no correl')
plt.xlabel('no correlation')
plt.grid(True)
plt.legend()
ax1 = plt.subplot(1,2,2)
#alpha透明度 edgecolors边缘颜色 label图例（结合legend使用）
plt.scatter(x,y2,color='g',alpha=.3,edgecolors='gray',label='correl')
plt.xlabel('correlation')
plt.grid(True)
plt.legend()
plt.show()

11.直方图和密度图

from sklearn.datasets import load_iris
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
#设置生成图形显示中文及负号
plt.rcParams['font.sans-serif']=['SimHei']
plt.rcParams['axes.unicode_minus'] = False
sns.set(color_codes = True)
#获取数据
iris = load_iris()
df = pd.DataFrame(iris.data,columns = iris.feature_names )
df['target'] = iris. target
print (df.head())
#数据分布可视化
sns.distplot(df['petal length (cm)'],bins=15)
plt.show()

12.散点图和直方图

from sklearn.datasets import load_iris
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
#设置生成图形显示中文及负号
plt.rcParams['font.sans-serif']=['SimHei']
plt.rcParams['axes.unicode_minus'] = False
sns.set(color_codes = True)
#获取数据
iris = load_iris()
df = pd.DataFrame(iris.data,columns = iris.feature_names )
df['target'] = iris. target
print (df.head())
#数据分布可视化
#jointplot()绘制散点图和直方图
sns.jointplot(x='petal length (cm)',y='petal width (cm)',data=df,size=8)
plt.show()

13.分组散点图

from sklearn.datasets import load_iris
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
#设置生成图形显示中文及负号
plt.rcParams['font.sans-serif']=['SimHei']
plt.rcParams['axes.unicode_minus'] = False
sns.set(color_codes = True)
#获取数据
iris = load_iris()
df = pd.DataFrame(iris.data,columns = iris.feature_names )
df['target'] = iris. target
print (df.head())
#数据分布可视化
#FacetGrid()进行分组
sns.FacetGrid(df,hue='target',size=8).map(plt.scatter,'sepal length (cm)','sepal width (cm)').add_legend()
plt.show()

14.六边形图

from sklearn.datasets import load_iris
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
#设置生成图形显示中文及负号
plt.rcParams['font.sans-serif']=['SimHei']
plt.rcParams['axes.unicode_minus'] = False
sns.set(color_codes = True)
#获取数据
iris = load_iris()
df = pd.DataFrame(iris.data,columns = iris.feature_names )
df['target'] = iris. target
print (df.head())
#数据分布可视化
#六边形图
sns.axes_style('white')
sns.jointplot(x='sepal length (cm)',y='sepal width (cm)',data=df,kind='hex',color='g')
plt.show()

15.二维核密度图

from sklearn.datasets import load_iris
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
#设置生成图形显示中文及负号
plt.rcParams['font.sans-serif']=['SimHei']
plt.rcParams['axes.unicode_minus'] = False
sns.set(color_codes = True)
#获取数据
iris = load_iris()
df = pd.DataFrame(iris.data,columns = iris.feature_names )
df['target'] = iris. target
print (df.head())
#数据分布可视化
#二维核密度估计图
g=sns.jointplot(x='sepal length (cm)',y='sepal width (cm)',data=df,kind='kde',color='m')
g.plot_joint(plt.scatter,c='w',s=30,linewidth=1,marker='*')
g.ax_joint.collections[0].set_alpha(0)
plt.show()

16.矩阵散点图

from sklearn.datasets import load_iris
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
#设置生成图形显示中文及负号
plt.rcParams['font.sans-serif']=['SimHei']
plt.rcParams['axes.unicode_minus'] = False
sns.set(color_codes = True)
#获取数据
iris = load_iris()
df = pd.DataFrame(iris.data,columns = iris.feature_names )
df['target'] = iris. target
print (df.head())
#数据分布可视化
#矩阵散点图
#纯散点图
g=sns.PairGrid(df)
g.map(plt.scatter)
plt.show()
#对角上绘制不同的函数
g=sns.PairGrid(df)
g.map_diag(plt.hist)
g.map_offdiag(plt.scatter)
plt.show()

 #单独分类变量控制
g=sns.PairGrid(df,hue='target')
g.map_diag(plt.hist)
g.map_offdiag(plt.scatter)
g.add_legend()
plt.show()
#特定关系
g=sns.PairGrid(df,vars=['sepal length (cm)','sepal width (cm)'],hue='target')
g.map_diag(plt.hist)
g.map_offdiag(plt.scatter)
plt.show()
#g=sns.PairGrid(df)
#g.map_upper(plt.scatter)
#g.map_lower(sns.kdeplot,camp='Blues_d')
#g.map_diag(sns.kdeplot,lw=3,legend=False)
sns.pairplot(df,hue='target',size=2.5)
plt.show()

17.小提琴图

from sklearn.datasets import load_iris
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
#设置生成图形显示中文及负号
plt.rcParams['font.sans-serif']=['SimHei']
plt.rcParams['axes.unicode_minus'] = False
sns.set(color_codes = True)
#获取数据
iris = load_iris()
df = pd.DataFrame(iris.data,columns = iris.feature_names )
df['target'] = iris. target
print (df.head())
#数据分布可视化
#小提琴图
sns.violinplot(x='target',y='sepal length (cm)',data=df,inner=None)
sns.swarmplot(x='target',y='sepal length (cm)',data=df,color='w',alpha=0.5)
sns.boxplot(x='target',y='sepal width (cm)',data=df)
plt.show()

18.绘制箱线图

from sklearn.datasets import load_iris
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np
#设置生成图形显示中文及负号
plt.rcParams['font.sans-serif']=['SimHei']
plt.rcParams['axes.unicode_minus'] = False
sns.set(color_codes = True)
#获取数据
iris = load_iris()
df = pd.DataFrame(iris.data,columns = iris.feature_names )
df['target'] = iris. target
print (df.head())
#数据分布可视化
#绘制箱线图
df.boxplot(by='target',figsize=(12,6))
plt.show()

19.K-means

# 数据标准化
scaler = StandardScaler()
X = df[['AVG_INTERVAL', 'LAST_TO_END', 'SUM_YR']]
X_scaled = scaler.fit_transform(X)

# K-means 聚类
k = 8  # 设置聚类的簇数
kmeans = KMeans(n_clusters=k, random_state=0)
kmeans.fit(X_scaled)
labels = kmeans.labels_

# 可视化聚类结果
plt.scatter(X_scaled[:, 0], X_scaled[:, 1], c=labels, cmap='viridis')
plt.scatter(kmeans.cluster_centers_[:, 0], kmeans.cluster_centers_[:, 1], marker='x', color='r', s=200)
plt.xlabel('Avg_Interval (Standardized)')
plt.ylabel('Last_to_End (Standardized)')
plt.title('K-means Clustering')
plt.show()


df = read_csv(r"/Users/y/Desktop/UCI_Credit_Card.csv",sep = ',',encoding='utf8')
print("源数据：")
print(df)
print("SUM_YR和LAST_TO_END之间的相关系数：")
df['LAST_TO_END'].corr(df['SUM_YR'])
print(df['LAST_TO_END'].corr(df['AVG_INTERVAL']))
print("三门成绩的相关系数：")
df.loc[:,['AVG_INTERVAL','SUM_YR_1','LAST_TO_END']].corr()
print(df.loc[:,['SUM_YR','SUM_YR_1','SUM_YR']].corr())

 

以上画图部分，把数据来源改下就行，大家尽情产图，尽情灌水吧！！！！！