Passing extra arguments to a custom scoring function in sklearn pipeline

Question

I need to perform univariate feature selection in sklearn with a custom score, therefore I am using GenericUnivariateSelect. However, as in documentation,

modes for selectors : {‘percentile’, ‘k_best’, ‘fpr’, ‘fdr’, ‘fwe’}

In my case, I needed to select features where the score was above a certain value, so I have implemented:

from sklearn.feature_selection.univariate_selection import _clean_nans
from sklearn.feature_selection.univariate_selection import f_classif                        
import numpy as np
import pandas as pd
from  sklearn.feature_selection import GenericUnivariateSelect
from sklearn.metrics import make_scorer 
from sklearn.feature_selection.univariate_selection import _BaseFilter
from sklearn.pipeline import Pipeline 



class SelectMinScore(_BaseFilter):
    # Sklearn documentation: modes for selectors : {‘percentile’,     ‘k_best’, ‘fpr’, ‘fdr’, ‘fwe’}
    # custom selector: 
    # select features according to the k highest scores.
    def __init__(self, score_func=f_classif, minScore=0.7):
        super(SelectMinScore, self).__init__(score_func)
        self.minScore = minScore
        self.score_func=score_func
    [...]
    def _get_support_mask(self):
        check_is_fitted(self, 'scores_')

        if self.minScore == 'all':
            return np.ones(self.scores_.shape, dtype=bool)
        else:
            scores = _clean_nans(self.scores_)
            mask = np.zeros(scores.shape, dtype=bool)

            # Custom part
            # only score above the min
            mask=scores>self.minScore
            if not np.any(mask):
                mask[np.argmax(scores)]=True
            return mask

However, I also need to use a custom score function which must receive extra arguments (XX) here: Unfortunatley, I could not solve using make_scorer

def Custom_Score(X,Y,XX):
      return 1

class myclass():
    def mymethod(self,_XX):

            custom_filter=GenericUnivariateSelect(Custom_Score(XX=_XX),mode='MinScore',param=0.7)   
        custom_filter._selection_modes.update({'MinScore': SelectMinScore})
        MyProcessingPipeline=Pipeline(steps=[('filter_step', custom_filter)])
    # finally
        X=pd.DataFrame(data=np.random.rand(500,3))
        y=pd.DataFrame(data=np.random.rand(500,1))
        MyProcessingPipeline.fit(X,y)
        MyProcessingPipeline.transform(X,y)

_XX=np.random.rand(500,1
C=myclass()
C.mymethod(_XX)

This raises the following error:

Traceback (most recent call last):

 File "<ipython-input-37-f493745d7e1b>", line 1, in <module>
runfile('C:/Users/_____/Desktop/pd-sk-integration.py', wdir='C:/Users/_____/Desktop')
File "C:\Users\______\AppData\Local\Continuum\Anaconda2\lib\site-packages\spyder\utils\site\sitecustomize.py", line 866, in runfile
execfile(filename, namespace)
File "C:\Users\\______\\AppData\Local\Continuum\Anaconda2\lib\site-packages\spyder\utils\site\sitecustomize.py", line 87, in execfile
exec(compile(scripttext, filename, 'exec'), glob, loc)=
File "C:/Users/______/Desktop/pd-sk-integration.py", line 65, in <module>
C.mymethod()
File "C:/Users/______/Desktop/pd-sk-integration.py", line 55, in mymethod
         custom_filter=GenericUnivariateSelect(Custom_Score(XX=_XX),mode='MinScore',param=0.7)
TypeError: Custom_Score() takes exactly 3 arguments (1 given)

EDIT:

I have tried a fix by adding an extra kwarg (XX) to the fit() of my SelectMinScore function, and by passing it as a fit paramter. As suggested by @TomDLT,

custom_filter = SelectMinScore(minScore=0.7)
pipe = Pipeline(steps=[('filter_step', custom_filter)])
pipe.fit(X,y, filter_step__XX=XX)

However, if I do

line 291, in set_params
(key, self.__class__.__name__))
ValueError: Invalid parameter XX for estimator   SelectMinScore. Check the list of available parameters with `estimator.get_params().keys()`.

Answer 1

As you can see in the code, the scorer function is not called with extra arguments, so there is currently no easy way in scikit-learn to pass your samples properties XX.

For your problem, a slightly hackish way could be to change the function fit in SelectMinScore, adding an additional parameter XX:

def fit(self, X, y, XX):
    """...""" 
    X, y = check_X_y(X, y, ['csr', 'csc'], multi_output=True)

    if not callable(self.score_func):
        raise TypeError("The score function should be a callable, %s (%s) "
                        "was passed."
                        % (self.score_func, type(self.score_func)))

    self._check_params(X, y)
    score_func_ret = self.score_func(X, y, XX)
    if isinstance(score_func_ret, (list, tuple)):
        self.scores_, self.pvalues_ = score_func_ret
        self.pvalues_ = np.asarray(self.pvalues_)
    else:
        self.scores_ = score_func_ret
        self.pvalues_ = None

    self.scores_ = np.asarray(self.scores_)

    return self

Then you could call the pipeline with extra fit params:

custom_filter = SelectMinScore(minScore=0.7)
pipe = Pipeline(steps=[('filter_step', custom_filter)])
pipe.fit(X,y, filter_step__XX=XX)