机器学习——13-垃圾邮件分类2

机器学习——13-垃圾邮件分类2 1.读取

file_path=r'C:\Users\AAAA\PycharmProjects\untitled\data\SMSSpamCollection' #读取⽂件

sms=open(file_path,'r',encoding='utf-8') #打开⽂件

sms_data=[]

sms_label=[]

csv_reader=csv.reader(sms,delimiter='\t') #读取⽂件

for line in csv_reader: #对每封邮件做预处理

sms_label.append(line[0]) #第⼀个字段存为⼀个类别，标签

sms_data.append(preprocessing(line[1])) #第⼆个字段存为⼀个类别，邮件内容

sms.close()

print("邮件类别:",sms_label)

print("处理后的邮件内容:",sms_data)

2.数据预处理

注意：preprocessed_text = ' '.join(tokens) 引号⾥⾯要加空格

import csv

import nltk

from nltk.corpus import stopwords

from nltk.stem import WordNetLemmatizer

def get_wordnet_pos(treebank_tag):#根据词性，⽣成还原参数pos

if treebank_tag.startswith('J'):

return nltk.corpus.wordnet.ADJ

elif treebank_tag.startswith('V'):

return nltk.corpus.wordnet.VERB

elif treebank_tag.startswith('N'):

return nltk.corpus.wordnet.NOUN

elif treebank_tag.startswith('R'):

return nltk.corpus.wordnet.ADV

else:

return nltk.corpus.wordnet.NOUN

def preprocessing(text): #预处理

tokens = [word for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]#分词

print("去掉停⽤词前数据长度：", len(tokens))

stops = stopwords.words("english")#停⽤词

tokens = [token for token in tokens if token not in stops]#

print("去掉停⽤词后数据长度：",len(tokens))

tokens = [token.lower() for token in tokens if len(token) >= 3]#将⼤写字母变为⼩写

lemmatizer = WordNetLemmatizer() #构建词性转换器

tag = nltk.pos_tag(tokens) #

词性标注

tokens = [lemmatizer.lemmatize(token, pos=get_wordnet_pos(tag[i][1])) for i, token in enumerate(tokens)]

preprocessed_text = ' '.join(tokens)

return preprocessed_text #返回处理结果

3.数据划分—训练集和测试集数据划分

from sklearn.model_selection import train_test_split

x_train,x_test, y_train, y_test = train_test_split(data, target, test_size=0.2, random_state=0, stratify=y_train) from sklearn.model_selection import train_test_split

x_train, x_test, y_train, y_test = train_test_split(sms_data, sms_label, test_size=0.2,

random_state=0,stratify=sms_label)

print('原数据长度:', len(sms_data),

'\n训练数据长度:', len(x_train),

'\n测试数据长度:', len(x_test))

4.⽂本特征提取

sklearn.feature_extraction.text.CountVectorizer

sklearn.feature_extraction.text.TfidfVectorizer

from sklearn.feature_extraction.text import TfidfVectorizer

tfidf2 = TfidfVectorizer()

#向量化

from sklearn.feature_extraction.text import TfidfVectorizer

tfidf2 = TfidfVectorizer()

X_train = tfidf2.fit_transform(x_train)

X_test = tfidf2.transform(x_test)

print('邮件以及向量关系数组：\n', X_train.toarray())

print('X_train矩阵：', X_train.toarray().shape, '\nX_test矩阵：', X_test.toarray().shape) #第⼀个数为邮件数，第⼆个数为单词数print('词汇表：\n', tfidf2.vocabulary_) #第⼀个为单词，第⼆个为该单词下标

观察邮件与向量的关系