场景化应用实战系列五：互联网舆情检测-CSDN博客

import requests
from bs4 import BeautifulSoup
import re

# 从网页收集评论数据
def scrape_comments(url):
    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'html.parser')
    comments = soup.find_all('div', class_='comment-content')
    return [comment.get_text() for comment in comments]

# 示例：爬取某网页的评论
url = 'https://example.com/comments'
comments = scrape_comments(url)

# 文本预处理
import jieba
import jieba.analyse
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split

# 分词和去除停用词
stopwords = set(line.strip() for line in open('stopwords.txt', 'r', encoding='utf-8').readlines())
processed_comments = []
for comment in comments:
    # 分词
    segs = jieba.lcut(comment)
    # 去除停用词和非中文字符
    filtered_segs = [seg for seg in segs if seg not in stopwords and re.match(r'^[\u4e00-\u9fa5]+$', seg)]
    processed_comments.append(' '.join(filtered_segs))

# 划分训练集和测试集
X_train, X_test = train_test_split(processed_comments, test_size=0.2, random_state=42)

2. 情感分析模型训练

from sklearn.svm import SVC
from sklearn.metrics import classification_report
from sklearn.preprocessing import LabelEncoder

# 加载预训练的词向量模型（示例使用自训练的TF-IDF向量化）
vectorizer = TfidfVectorizer(max_features=5000)
X_train_vec = vectorizer.fit_transform(X_train)
X_test_vec = vectorizer.transform(X_test)

# 假设我们已经有了情感标签
y_train = [...]  # 训练集情感标签
y_test = [...]   # 测试集情感标签

# 编码标签
label_encoder = LabelEncoder()
y_train_encoded = label_encoder.fit_transform(y_train)
y_test_encoded = label_encoder.transform(y_test)

# 训练分类器
clf = SVC(kernel='linear', C=1.0, probability=True)
clf.fit(X_train_vec, y_train_encoded)

# 模型评估
y_pred = clf.predict(X_test_vec)
print(classification_report(y_test_encoded, y_pred, target_names=label_encoder.classes_))

3. 舆情数据存储

import sqlite3

# 创建数据库连接
conn = sqlite3.connect('sentiment_analysis.db')
cursor = conn.cursor()

# 创建表
cursor.execute('''
CREATE TABLE IF NOT EXISTS sentiment_data (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    text TEXT NOT NULL,
    sentiment TEXT NOT NULL,
    timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
)
''')

# 插入数据
def store_sentiment_data(text, sentiment):
    cursor.execute('INSERT INTO sentiment_data (text, sentiment) VALUES (?, ?)', (text, sentiment))
    conn.commit()

# 示例：存储分析结果
for text, sentiment in zip(X_test, y_test):
    store_sentiment_data(text, sentiment)

# 查询数据
cursor.execute('SELECT * FROM sentiment_data')
rows = cursor.fetchall()
for row in rows:
    print(row)

# 关闭数据库连接
conn.close()

4. 后端部署与服务化

from flask import Flask, request, jsonify
import joblib
import numpy as np

# 加载模型和向量化器
model = joblib.load('sentiment_model.pkl')
vectorizer = joblib.load('tfidf_vectorizer.pkl')
label_encoder = joblib.load('label_encoder.pkl')

app = Flask(__name__)

@app.route('/analyze', methods=['POST'])
def analyze_sentiment():
    data = request.json
    text = data.get('text', '')
    
    # 文本预处理（分词、去停用词等步骤应在此处实现）
    # 为了简化，这里假设直接使用向量化器转换文本
    text_vec = vectorizer.transform([text])
    
    # 情感预测
    sentiment_prob = model.predict_proba(text_vec)[0]
    sentiment_pred = model.predict(text_vec)[0]
    sentiment_label = label_encoder.inverse_transform([sentiment_pred])[0]
    
    # 存储数据
    store_sentiment_data(text, sentiment_label)