Python Exception: Data must be 1-dimensional

Question

I have a function findMaxEval which I invoke in a following way:

eMax0,var0=findMaxEval(np.diag(eVal0),q,bWidth=.01)

where np.diag(eVal0) is an ndarray of shape (1000,), q is a number (10).

findMaxEval has the following definition:

def findMaxEval(eVal,q,bWidth):
    out=minimize(lambda *x:errPDFs(*x),.5,args= (eVal,q,bWidth),bounds=((1E-5,1-1E-5),))
    if out['success']:var=out['x'][0]
    else:var=1
    eMax=var*(1+(1./q)**.5)**2
    return eMax,var

This funtion tries to minimize errPDFs which is defined as follows:

def errPDFs(var,eVal,q,bWidth,pts=1000):
    pdf0=mpPDF(var,q,pts)
    pdf1=fitKDE(eVal,bWidth,x=pdf0.index.values)
    sse=np.sum((pdf1-pdf0)**2)
    return sse

var is a number which I pass in the findMaxEval function in minimize, initial value is 0.5.

Further, mpPDF and fitKDE are defined:

def mpPDF(var,q,pts):
    eMin,eMax=var*(1-(1./q)**.5)**2,var*(1+(1./q)**.5)**2
    eVal=np.linspace(eMin,eMax,pts)
    pdf=q/(2*np.pi*var*eVal)*((eMax-eVal)*(eVal-eMin))**.5
    pdf=pd.Series(pdf,index=eVal)
    return pdf

def fitKDE(obs,bWidth=.25,kernel='gaussian',x=None):
    if len(obs.shape)==1:obs=obs.reshape(-1,1)
    kde=KernelDensity(kernel=kernel,bandwidth=bWidth).fit(obs)
    if x is None:x=np.unique(obs).reshape(-1,1)
    if len(x.shape)==1:x=x.reshape(-1,1)
    logProb=kde.score_samples(x) # log(density)
    pdf=pd.Series(np.exp(logProb),index=x.flatten())
    return pdf

When I invoke findMaxEval (first line in the description), I get the following error:

---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
<ipython-input-25-abd7cf64e843> in <module>
----> 1 eMax0,var0=findMaxEval(np.diag(eVal0),q,bWidth=.01)
      2 nFacts0=eVal0.shape[0]-np.diag(eVal0)[::-1].searchsorted(eMax0)

<ipython-input-24-f44a1e9d84b1> in findMaxEval(eVal, q, bWidth)
      1 def findMaxEval(eVal,q,bWidth):
      2     # Find max random eVal by fitting Marcenko’s dist
----> 3     out=minimize(lambda *x:errPDFs(*x),.5,args= (eVal,q,bWidth),bounds=((1E-5,1-1E-5),))
      4     if out['success']:var=out['x'][0]
      5     else:var=1

/opt/anaconda3/lib/python3.7/site-packages/scipy/optimize/_minimize.py in minimize(fun, x0, args, method, jac, hess, hessp, bounds, constraints, tol, callback, options)
    598         return _minimize_neldermead(fun, x0, args, callback, **options)
    599     elif meth == 'powell':
--> 600         return _minimize_powell(fun, x0, args, callback, **options)
    601     elif meth == 'cg':
    602         return _minimize_cg(fun, x0, args, jac, callback, **options)

/opt/anaconda3/lib/python3.7/site-packages/scipy/optimize/lbfgsb.py in _minimize_lbfgsb(fun, x0, args, jac, bounds, disp, maxcor, ftol, gtol, eps, maxfun, maxiter, iprint, callback, maxls, **unknown_options)
    333 
    334     while 1:
--> 335         # x, f, g, wa, iwa, task, csave, lsave, isave, dsave = \
    336         _lbfgsb.setulb(m, x, low_bnd, upper_bnd, nbd, f, g, factr,
    337                        pgtol, wa, iwa, task, iprint, csave, lsave,

/opt/anaconda3/lib/python3.7/site-packages/scipy/optimize/lbfgsb.py in func_and_grad(x)
    278     # unbounded variables must use None, not +-inf, for optimizer to work properly
    279     bounds = [(None if l == -np.inf else l, None if u == np.inf else u) for l, u in bounds]
--> 280 
    281     if disp is not None:
    282         if disp == 0:

/opt/anaconda3/lib/python3.7/site-packages/scipy/optimize/optimize.py in function_wrapper(*wrapper_args)
    324 
    325     def function_wrapper(*wrapper_args):
--> 326         ncalls[0] += 1
    327         return function(*(wrapper_args + args))
    328 

<ipython-input-24-f44a1e9d84b1> in <lambda>(*x)
      1 def findMaxEval(eVal,q,bWidth):
      2     # Find max random eVal by fitting Marcenko’s dist
----> 3     out=minimize(lambda *x:errPDFs(*x),.5,args= (eVal,q,bWidth),bounds=((1E-5,1-1E-5),))
      4     if out['success']:var=out['x'][0]
      5     else:var=1

<ipython-input-23-24070a331535> in errPDFs(var, eVal, q, bWidth, pts)
      1 def errPDFs(var,eVal,q,bWidth,pts=1000):
      2     # Fit error
----> 3     pdf0=mpPDF(var,q,pts) # theoretical pdf
      4     pdf1=fitKDE(eVal,bWidth,x=pdf0.index.values) # empirical pdf
      5     sse=np.sum((pdf1-pdf0)**2)

<ipython-input-17-565d70018af2> in mpPDF(var, q, pts)
     10     eVal=np.linspace(eMin,eMax,pts)
     11     pdf=q/(2*np.pi*var*eVal)*((eMax-eVal)*(eVal-eMin))**.5
---> 12     pdf=pd.Series(pdf,index=eVal)
     13     return pdf

/opt/anaconda3/lib/python3.7/site-packages/pandas/core/series.py in __init__(self, data, index, dtype, name, copy, fastpath)
    312 
    313     def _init_dict(self, data, index=None, dtype=None):
--> 314         """
    315         Derive the "_data" and "index" attributes of a new Series from a
    316         dictionary input.

/opt/anaconda3/lib/python3.7/site-packages/pandas/core/internals/construction.py in sanitize_array(data, index, dtype, copy, raise_cast_failure)

Exception: Data must be 1-dimensional

I don't understand what should be 1-Dimensional. np.diag(eVal0) is of shape (1000,).

I looked at all the other similar questions, but none seems to help me solve this.

Thanks.

Answer 1

The error is unrelated to bounds.

For some reason minimize() calls the custom function errPDFs() with the argument to be optimized - minimize() calls this for x0 - which is an array. So if you redefine the function errPDFs() to extract the first element of an array:

def errPDFs(var, eVal, q, bWidth, pts=1000):
    print("var:"+var)
    pdf0 = mpPDF(var[0], q, pts) #theoretical pdf
    pdf1 = fitKDE(eVal, bWidth, x=pdf0.index.values) #empirical pdf
    sse = np.sum((pdf1-pdf0)**2)
    print("sse:"+str(sse))
    return sse 

It should work.

Sample output:

>>> out = minimize(lambda *x: errPDFs(*x), .5, args=(eVal, q, bWidth),bounds= 
 ((1E-5, 1-1E-5),))
 var:[0.5]
 sse:743.6200749295413
 var:[0.50000001]
 sse:743.6199819531047
 var:[0.99999]
 sse:289.1462047531385
 ...

Answer 2

Updated 6/29 ... I got it to run this way, which is strange because it is the same thing, must be a bug in the library or casting explicitly like this gets it into the precise format desired:

import numpy as np
import pandas as pd
from scipy.optimize import minimize
from sklearn.neighbors import KernelDensity

def findMaxEval(eVal, q, bWidth):
    bnds = ((float(1e5/10000000000), float(0.99999*-1)),)

    print(bnds)
    out = minimize(lambda *x: errPDFs(*x), .5, args=(eVal, q, bWidth), bounds=bnds)
    if out['success']: var = out['x'][0]
    else: var = 1
    eMax = var*(1+(1./q)**.5)**2
    return eMax, var

def errPDFs(var, eVal, q, bWidth, pts = 1000):
    pdf0 = mpPDF(var, q, pts)
    pdf1 = fitKDE(eVal, bWidth, x=pdf0.index.values)
    sse=np.sum((pdf1-pdf0)**2)
    return sse

def mpPDF(var, q, pts):
    eMin, eMax=var*(1-(1./q)**.5)**2,var*(1+(1./q)**.5)**2
    eVal = np.linspace(eMin, eMax, pts)
    pdf = q/(2*np.pi*var*eVal)*((eMax-eVal)*(eVal-eMin))**.5
    pdf = pd.Series(pdf, index=eVal)
    return pdf

def fitKDE(obs, bWidth = .25, kernel='gaussian', x=None):
    if len(obs.shape) == 1: obs = obs.reshape(-1, 1)
    kde=KernelDensity(kernel=kernel, bandwidth=bWidth).fit(obs)
    if x is None: x = np.unique(obs).reshape(-1, 1)
    if len(x.shape) == 1: x = x.reshape(-1, 1)
    logProb = kde.score_samples(x)
    pdf=pd.Series(np.exp(logProb), index=x.flatten())
    return pdf

eMax0, var0 = findMaxEval((1000,), 10, bWidth=.01)

print(eMax0)
print(var0)

Here is the updated output on Macbook in PyCharm Community, Python version 3.8.1: