如何训练 n-gram 的朴素贝叶斯分类器 (movie_reviews)

Question

以下是模型数据集Naive Bayes Classifier训练的代码。我想通过考虑模型来训练和分析其性能。我们怎样才能做到呢。movie_reviewsunigrambigramtrigram

import nltk.classify.util
from nltk.classify import NaiveBayesClassifier
from nltk.corpus import movie_reviews
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize

def create_word_features(words):
    useful_words = [word for word in words if word not in stopwords.words("english")] 
    my_dict = dict([(word, True) for word in useful_words])
    return my_dict

pos_data = []
for fileid in movie_reviews.fileids('pos'):
    words = movie_reviews.words(fileid)
    pos_data.append((create_word_features(words), "positive"))    

neg_data = []
for fileid in movie_reviews.fileids('neg'):
    words = movie_reviews.words(fileid)
    neg_data.append((create_word_features(words), "negative")) 

train_set = pos_data[:800] + neg_data[:800]
test_set =  pos_data[800:] + neg_data[800:]

classifier = NaiveBayesClassifier.train(train_set)

accuracy = nltk.classify.util.accuracy(classifier, test_set)

Answer 1

只需更改您的特征器

from nltk import ngrams

def create_ngram_features(words, n=2):
    ngram_vocab = ngrams(words, n)
    my_dict = dict([(ng, True) for ng in ngram_vocab])
    return my_dict

顺便说一句，如果您更改特征器以对停用词列表使用一组并仅初始化一次，您的代码将会快得多。

stoplist = set(stopwords.words("english"))

def create_word_features(words):
    useful_words = [word for word in words if word not in stoplist] 
    my_dict = dict([(word, True) for word in useful_words])
    return my_dict

有人真的应该告诉 NLTK 人员将停用词列表转换为集合类型，因为它“从技术上讲”是一个唯一的列表（即集合）。

>>> from nltk.corpus import stopwords
>>> type(stopwords.words('english'))
<class 'list'>
>>> type(set(stopwords.words('english')))
<class 'set'>

为了享受基准测试的乐趣

import nltk.classify.util
from nltk.classify import NaiveBayesClassifier
from nltk.corpus import movie_reviews
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk import ngrams

def create_ngram_features(words, n=2):
    ngram_vocab = ngrams(words, n)
    my_dict = dict([(ng, True) for ng in ngram_vocab])
    return my_dict

for n in [1,2,3,4,5]:
    pos_data = []
    for fileid in movie_reviews.fileids('pos'):
        words = movie_reviews.words(fileid)
        pos_data.append((create_ngram_features(words, n), "positive"))    

    neg_data = []
    for fileid in movie_reviews.fileids('neg'):
        words = movie_reviews.words(fileid)
        neg_data.append((create_ngram_features(words, n), "negative")) 

    train_set = pos_data[:800] + neg_data[:800]
    test_set =  pos_data[800:] + neg_data[800:]

    classifier = NaiveBayesClassifier.train(train_set)

    accuracy = nltk.classify.util.accuracy(classifier, test_set)
    print(str(n)+'-gram accuracy:', accuracy)

[出去]：

1-gram accuracy: 0.735
2-gram accuracy: 0.7625
3-gram accuracy: 0.8275
4-gram accuracy: 0.8125
5-gram accuracy: 0.74

您的原始代码返回的精度为 0.725。

使用更多 ngram 阶数

import nltk.classify.util
from nltk.classify import NaiveBayesClassifier
from nltk.corpus import movie_reviews
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk import everygrams

def create_ngram_features(words, n=2):
    ngram_vocab = everygrams(words, 1, n)
    my_dict = dict([(ng, True) for ng in ngram_vocab])
    return my_dict

for n in range(1,6):
    pos_data = []
    for fileid in movie_reviews.fileids('pos'):
        words = movie_reviews.words(fileid)
        pos_data.append((create_ngram_features(words, n), "positive"))    

    neg_data = []
    for fileid in movie_reviews.fileids('neg'):
        words = movie_reviews.words(fileid)
        neg_data.append((create_ngram_features(words, n), "negative")) 

    train_set = pos_data[:800] + neg_data[:800]
    test_set =  pos_data[800:] + neg_data[800:]
    classifier = NaiveBayesClassifier.train(train_set)

    accuracy = nltk.classify.util.accuracy(classifier, test_set)
    print('1-gram to', str(n)+'-gram accuracy:', accuracy)

[出去]：

1-gram to 1-gram accuracy: 0.735
1-gram to 2-gram accuracy: 0.7625
1-gram to 3-gram accuracy: 0.7875
1-gram to 4-gram accuracy: 0.8
1-gram to 5-gram accuracy: 0.82