这边有个需求,需要把训练好的模型存进db里面,之前从来没存过都是直接用pickle或者joblib存到本地,就查了下,记录。
中心实现是把joblib变成二进制存进db里面
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.cluster import AgglomerativeClustering
from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import train_test_split
from sqlalchemy import create_engine, MetaData, Table, insert,update,select
from joblib import dump, load
from io import BytesIO
df=pd.read_csv('./oldDataTotal.csv')
processContent=df['processed_content']
X_train, X_test = train_test_split(processContent.tolist(),
test_size=0.5,
random_state=42)
cv=CountVectorizer()
cv_x=cv.fit_transform(X_train)
y_label = AgglomerativeClustering(n_clusters=None,
distance_threshold=3.1,
compute_distances=True).fit_predict(
cv_x.toarray())
mlb=MultinomialNB()
mlb.fit(cv_x,y_label)
pd.DataFrame(list(zip(X_train,y_label)),columns=['text','predict']).to_csv('./train.csv',index=False)
dump(mlb, 'filename.joblib')
engine=create_engine('mysql+pymysql://root:xxxxx@localhost:3306/testdb',future=True)
metadata = MetaData()
table = Table('model', metadata, autoload_with=engine)
with open('filename.joblib', 'rb') as f:
model_data = f.read()
stmt = update(table).values(id=1,model=model_data)
with engine.connect() as conn:
result = conn.execute(stmt)
conn.commit()
stmt = select(table).where(table.c.id == 1)
with engine.connect() as conn:
result = conn.execute(stmt)
row = result.fetchone()
model_data = row['model']
model_file = BytesIO(model_data)
loaded_model = load(model_file)
cv_x_test=cv.transform(X_test)
y_pred = loaded_model.predict(cv_x_test)
pd.DataFrame(list(zip(X_test,y_pred)),columns=['text','predict']).to_csv('./pred.csv',index=False)
这边用的是mysql做的例子,其他数据库就是db链接变一下就行了,db里面model里面的字段是longblob类型。