【Python】文本词频统计

词频统计"/>

【Python】文本词频统计

哈姆雷特英文

 .txt

三国演义中文

.txt

 哈姆雷特英文词频分析

def getText():txt=open("hamlet.txt","r").read()#打开文本,输入具体的文本路径txt=txt.lower()#将文本中所有的英文字符变成小写for ch in '!"#$%&()*+,-./;:<=>?@[\\]^‘_{|}~':txt=txt.replace(ch," ")return txt #去掉特殊符号
hamletTxt=getText()#调用函数对文本进行处理
words=hamletTxt.split()#进行列表
counts={}#字典
for word in words:counts[word]=counts.get(word,0)+1#获取到的词在字典中寻找如果有的话在原来的基础上+1，如果没有就收录到字典中
items=list(counts.items())#变成列表类型
items.sort(key=lambda x:x[1],reverse=True)#对列表排序
for i in range(10):#将出现次数前10的单词输出并输出出现次数word,count=items[i]print("{0:<10}{1:>5}".format(word,count))

 三国演义人物出场次数

import jieba#引入jieba分词库
txt = open("threekingdoms.txt", "r", encoding="utf-8").read()#打开文本
words = jieba.lcut(txt)#进行分词处理并形成列表
counts = {}#构造字典，逐一遍历words中的中文单词进行处理，并用字典计数
for word in words:if len(word) == 1:continueelse:counts[word] = counts.get(word, 0) + 1
items = list(counts.items())#转换列表类型并排序
items.sort(key=lambda x:x[1], reverse=True)
for i in range(15):#输出前15位单词word, count = items[i]print("{0:<10}{1:<5}".format(word, count))

结果：

 上面有不是人物的词，需要改造

import jieba
txt = open("threekingdoms.txt", "r", encoding="utf-8").read()
excludes = {"将军", "却说", "荆州", "二人", "不可", "不能", "如此", "主公",\"军士", "商议", "如何", "左右", "军马", "引兵", "次日", "大喜",\"天下", "东吴", "于是", "今日", "不敢", "魏兵", "陛下", "一人",\"都督", "人马", "不知"}#排除不是人名的词汇，加到这个排除词库中
words = jieba.lcut(txt)
counts = {}
for word in words:#进行人名关联，防止重复if len(word) == 1:continueelif word == "诸葛亮" or word == "孔明曰":rword = "孔明"elif word == "关公" or word == "云长":rword = "关羽"elif word == "玄德" or word == "玄德曰":rword = "刘备"elif word == "孟德" or word == "丞相":rword = "曹操"else:rword = wordcounts[rword] = counts.get(rword, 0) + 1
for word in excludes:del counts[word]
items = list(counts.items())
items.sort(key=lambda x:x[1], reverse=True)
for i in range(10):word, count = items[i]print("{0:<10}{1:<5}".format(word, count))

结果

不断优化。。。。