每次迭代在多个CPU上训练不同的scikit-learn分类器

Question

我有一个脚本,随机生成一组数据并训练几个分类器,以相互比较它们(它非常类似于http://scikit-learn.org/stable/auto_examples/classification/plot_classifier_comparison.html):

from itertools import product

import numpy as np

from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.naive_bayes import GaussianNB, MultinomialNB
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis

from sklearn.datasets import make_classification
from sklearn.preprocessing import StandardScaler
from sklearn.cross_validation import train_test_split

names = ["Linear SVM", "Decision Tree",
     "Random Forest", "AdaBoost", "Naive Bayes", "Linear Discriminant Analysis",
     "Quadratic Discriminant Analysis"]

def griddy_mcsearchface(num_samples, num_feats, num_feats_to_remove): 
    classifiers = [
        SVC(kernel="linear", C=0.025),
        DecisionTreeClassifier(max_depth=5),
        RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1),
        AdaBoostClassifier(), GaussianNB(),
        LinearDiscriminantAnalysis(),
        QuadraticDiscriminantAnalysis()]

    classifiers2 = [
        SVC(kernel="linear", C=0.025),
        DecisionTreeClassifier(max_depth=5),
        RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1),
        AdaBoostClassifier(), GaussianNB(),
        LinearDiscriminantAnalysis(),
        QuadraticDiscriminantAnalysis()]

    X, y = make_classification(n_samples=num_samples, n_features=num_feats, n_redundant=0, n_informative=2,
                           random_state=1, n_clusters_per_class=1)
    X = StandardScaler().fit_transform(X)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2)

    for name, clf, clf2 in zip(names, classifiers, classifiers2):
        clf.fit(X_train, y_train)
        score = clf.score(X_test, y_test)
        # Remove 40% of the features.
        clf2.fit(X_train[:,:-num_feats_to_remove], y_train)
        score2 = clf2.score(X_test[:,:-num_feats_to_remove], y_test)
        yield (num_samples, num_feats, num_feats_to_remove, name, score, score2)

并运行它:

_samples = [100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000]
_feats = [10, 20, 50, 100, 200, 500, 10000]
_feats_to_rm = [5, 10, 25, 50, 100, 250]
for num_samples, num_feats, num_feats_to_remove in product(_samples, _feats, _feats_to_rm):
    if num_feats <= num_feats_to_remove:
        continue
    for i in griddy_mcsearchface(num_samples, num_feats, num_feats_to_remove):
        print (i)

该脚本输出如下内容:

(100, 10, 5, 'Linear SVM', 1.0, 0.40000000000000002)
(100, 10, 5, 'Decision Tree', 1.0, 0.65000000000000002)
(100, 10, 5, 'Random Forest', 1.0, 0.90000000000000002)
(100, 10, 5, 'AdaBoost', 1.0, 0.65000000000000002)
(100, 10, 5, 'Naive Bayes', 1.0, 0.75)
(100, 10, 5, 'Linear Discriminant Analysis', 1.0, 0.40000000000000002)
(100, 10, 5, 'Quadratic Discriminant Analysis', 1.0, 0.84999999999999998)
(100, 20, 5, 'Linear SVM', 1.0, 1.0)
(100, 20, 5, 'Decision Tree', 0.94999999999999996, 0.94999999999999996)
(100, 20, 5, 'Random Forest', 0.80000000000000004, 0.75)
(100, 20, 5, 'AdaBoost', 1.0, 0.94999999999999996)
(100, 20, 5, 'Naive Bayes', 1.0, 1.0)
(100, 20, 5, 'Linear Discriminant Analysis', 1.0, 1.0)
(100, 20, 5, 'Quadratic Discriminant Analysis', 0.84999999999999998, 0.94999999999999996)
(100, 20, 10, 'Linear SVM', 0.94999999999999996, 0.65000000000000002)
(100, 20, 10, 'Decision Tree', 0.94999999999999996, 0.59999999999999998)
(100, 20, 10, 'Random Forest', 0.75, 0.69999999999999996)
(100, 20, 10, 'AdaBoost', 0.94999999999999996, 0.69999999999999996)
(100, 20, 10, 'Naive Bayes', 0.94999999999999996, 0.75)

但clf.fit()现在是单线程的.

假设我有足够的线程来为每次迭代运行所有分类器, 我怎样才能在每次迭代时使用不同的线程训练分类器for num_samples, num_feats, num_feats_to_remove in product(_samples, _feats, _feats_to_rm)？

如果我被限制为4或8个线程,但我需要为每次迭代训练> 4或> 8个分类器,它是如何完成的？

Answer 1

这不是一个答案，而是对你的第一个问题的答案的粗略草图，

我如何能够使用不同的线程训练分类器 for every iteration of for num_samples, num_feats, num_feats_to_remove in product(_samples, _feats, _feats_to_rm)

我认为您的意思是，对于for name, clf, clf2 in zip(names, classifiers, classifiers2):您想要的每次迭代clf，并clf2在不同的处理器上进行训练。

这是一些工作代码作为起点（它实现得很差，但总体思路是存在的）：

from itertools import product

import numpy as np
import multiprocessing
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.naive_bayes import GaussianNB, MultinomialNB
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis

from sklearn.datasets import make_classification
from sklearn.preprocessing import StandardScaler
from sklearn.cross_validation import train_test_split

names = ["Linear SVM", "Decision Tree",
     "Random Forest", "AdaBoost", "Naive Bayes", "Linear Discriminant Analysis",
     "Quadratic Discriminant Analysis"]

# def mp_handler():
#     p = multiprocessing.Pool(8)
#     p.map(mp_worker, data) 

def mp_worker((name, clf, X_train, y_train, X_test, y_test, num_features_to_remove)):
    if num_features_to_remove == False:
        clf.fit(X_train, y_train)
        return ('score1', clf.score(X_test, y_test))

    clf.fit(X_train[:,:-num_feats_to_remove], y_train)
    return ('score2', clf.score(X_test[:,:-num_feats_to_remove], y_test))

def griddy_mcsearchface(num_samples, num_feats, num_feats_to_remove): 
    classifiers = [
        SVC(kernel="linear", C=0.025),
        DecisionTreeClassifier(max_depth=5),
        RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1),
        AdaBoostClassifier(), GaussianNB(),
        LinearDiscriminantAnalysis(),
        QuadraticDiscriminantAnalysis()]

    classifiers2 = [
        SVC(kernel="linear", C=0.025),
        DecisionTreeClassifier(max_depth=5),
        RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1),
        AdaBoostClassifier(), GaussianNB(),
        LinearDiscriminantAnalysis(),
        QuadraticDiscriminantAnalysis()]

    X, y = make_classification(n_samples=num_samples, n_features=num_feats, n_redundant=0, n_informative=2,
                           random_state=1, n_clusters_per_class=1)
    X = StandardScaler().fit_transform(X)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2)



    for name, clf, clf2 in zip(names, classifiers, classifiers2):

        p = multiprocessing.Pool(2) #set to 2 for using two processors; one processor per classfier
        #The integer parameter you pass to Pool is equal to the number of SETS of classifiers you have
        data = (name, clf, X_train, y_train, X_test, y_test, False), (name, clf, X_train, y_train, X_test, y_test, num_feats_to_remove)
        res =  p.map(mp_worker, data) #this splits the two classification tasks acrpss two separate processors
        for i,j in res: #parse the results
            if i == 'score1':
                score1 = j
            else:
                score2 = j

        yield (num_samples, num_feats, num_feats_to_remove, name, score1, score2)

if __name__ == '__main__':


    _samples = [100, 200]
    _feats = [10, 20]
    _feats_to_rm = [5, 10]
    for num_samples, num_feats, num_feats_to_remove in product(_samples, _feats, _feats_to_rm):
        if num_feats <= num_feats_to_remove:
            continue
        for i in griddy_mcsearchface(num_samples, num_feats, num_feats_to_remove):
            print (i)

如果我误解了您的问题，那么可以修改上面代码中的一般原则以满足您的需求。我从上面代码中接受的答案中得出。