import pandas as pd
import re
import nltk
from nltk import FreqDist
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from nltk.tokenize import word_tokenize
import spacy
from spacy.lang.en.stop_words import STOP_WORDS
from gensim.corpora import Dictionary
from gensim.models import LdaModel
# 下载NLTK的停用词、情感分析和词性标注所需的资源
nltk.download('stopwords')
nltk.download('punkt')
nltk.download('vader_lexicon')
# 加载SpaCy的英文NLP模型
nlp = spacy.load("en_core_web_sm")
# 读取Excel文件
df = pd.read_excel('nltk分词处理结果第二次.xlsx')
# 定义文本清洗函数
def clean_text(text):
# 去除HTML标签
cleaned_text = re.sub(r'<.*?>', '', text)
# 去除多余空格和换行符
cleaned_text = re.sub(r'\s+', ' ', cleaned_text)
# 转换为小写
cleaned_text = cleaned_text.lower()
return cleaned_text
# 清洗文本数据
df['cleaned_content'] = df['content'].apply(clean_text)
# 词频分析
words = []
for text in df['cleaned_content']:
words += word_tokenize(text)
freq_dist = FreqDist(words)
print("词频分析结果：", freq_dist.most_common(10))
# 情感分析
sia = SentimentIntensityAnalyzer()
df['sentiment_score'] = df['cleaned_content'].apply(lambda x: sia.polarity_scores(x)['compound'])
print("情感分析结果：", df['sentiment_score'])
# 定义阈值
positive_threshold = 0.5
negative_threshold = -0.5
# 根据情感分数进行分类
def classify_sentiment(score):
if score > positive_threshold:
return '积极'
elif score < negative_threshold:
return '消极'
else:
return '中性'
# 应用分类函数，创建新的列 'sentiment_category'
df['sentiment_category'] = df['sentiment_score'].apply(classify_sentiment)
# 输出带有情感分类的数据
print(df[['cleaned_content', 'sentiment_score', 'sentiment_category']])
# 主题建模
tokens = [[token.text.lower() for token in nlp(text) if token.is_alpha and token.text.lower() not in STOP_WORDS] for text in df['cleaned_content']]
dictionary = Dictionary(tokens)
corpus = [dictionary.doc2bow(text) for text in tokens]
lda_model = LdaModel(corpus, num_topics=5, id2word=dictionary, passes=15)
topics = lda_model.print_topics(num_words=5)
print("主题建模结果：")
for topic in topics:
print(topic)