Permutation Test

           

import uuid
import numpy as np
from sklearn.datasets import load_breast_cancer

from photonai.processing.permutation_test import PermutationTest


def create_hyperpipe():
    # this is needed here for the parallelisation
    from photonai.base import Hyperpipe, PipelineElement, OutputSettings
    from photonai.optimization import FloatRange, Categorical, IntegerRange
    from sklearn.model_selection import GroupKFold
    from sklearn.model_selection import KFold

    settings = OutputSettings(mongodb_connect_url='mongodb://localhost:27017/photon_results',
                              project_folder='./tmp/')
    my_pipe = Hyperpipe('permutation_test_1',
                        optimizer='grid_search',
                        metrics=['accuracy', 'precision', 'recall'],
                        best_config_metric='accuracy',
                        outer_cv=GroupKFold(n_splits=2),
                        inner_cv=KFold(n_splits=2),
                        calculate_metrics_across_folds=True,
                        eval_final_performance=True,
                        verbosity=1,
                        output_settings=settings)

    # Add transformer elements
    my_pipe += PipelineElement("StandardScaler", hyperparameters={},
                               test_disabled=True, with_mean=True, with_std=True)

    my_pipe += PipelineElement("PCA",  # hyperparameters={'n_components': IntegerRange(5, 15)},
                               test_disabled=False)

    # Add estimator
    my_pipe += PipelineElement("SVC", hyperparameters={'kernel': ['linear', 'rbf']}, #C': FloatRange(0.1, 5),
                               gamma='scale', max_iter=1000000)

    return my_pipe


X, y = load_breast_cancer(return_X_y=True)
my_perm_id = str(uuid.uuid4())
groups = np.random.random_integers(0, 3, (len(y), ))

# in case the permutation test for this specific hyperpipe has already been calculated, PHOTON will skip the permutation
# runs and load existing results
perm_tester = PermutationTest(create_hyperpipe, n_perms=2, n_processes=1, random_state=11,
                              permutation_id=my_perm_id)
perm_tester.fit(X, y, groups=groups)

results = PermutationTest._calculate_results(my_perm_id, mongodb_path='mongodb://localhost:27017/photon_results')
print(results.p_values)
Permutation Test

Acknowledgements
