I found similar issues around the internet but with slight differences and none of the solutions worked for me. I have a set of explanatory variables X (2085,12) and an explained variable y (2085,1) which I have to do some stuff on, including the use of these sklearn classes (as title). In order to get the right hyperparameters I have arranged the code as follows:
#solver: sgd
mlpsgd = MLPRegressor(max_iter = 1000, solver='sgd')
alpha = [float(x) for x in np.logspace(start = -6, stop = 3, num = 100)]
hidden_layer_sizes = [(int(x),int(y),int(z)) for x in np.logspace(start = 0, stop = 2.2, num = 8) for y in np.logspace(start = 0, stop = 2.2, num = 8) for z in np.logspace(start = 0, stop = 2.2, num = 8)]
hidden_layer_sizes.extend((int(x),int(y)) for x in np.logspace(start = 0, stop = 2, num = 25) for y in np.logspace(start = 0, stop = 2, num = 25))
hidden_layer_sizes.extend((int(x),) for x in np.logspace(start = 1, stop = 2, num = 1000))
activation = ['logistic', 'tanh', 'relu']
learning_rate = ['constant', 'invscaling','adaptive']
learning_rate_init = [float(x) for x in np.logspace(start = -5, stop = 0, num = 20)]
random_grid3 = {'learning_rate': learning_rate,'activation': activation,'learning_rate_init': learning_rate_init, 'hidden_layer_sizes': hidden_layer_sizes, 'alpha': alpha}
mlp_random3 = RandomizedSearchCV(estimator = mlpsgd, param_distributions = random_grid3, n_iter = 350, n_jobs=-1)
mlp_random3.fit(X, y)
Now I know that the whole random grid is insanely huge, but I tried even with a very small one and this is not the issue (and this way it is more of a match for the type of research I am supposed to do), and I should mention that I use windows and the program starts with
if __name__ == '__main__':
as I understood (hopefully correctly) it is needed for the multiprocessing I am asking in the second to last line of the firs part of code I attached. Well the thing is that when I run the code some of the 350 iteration are processed correctly, but then it stops and prints this error:
Traceback (most recent call last):
File "c:\Users\mat\OneDrive\Desktop\TES\Analisi\Tesi.py", line 164, in <module>
perc = mlpottimizzata(x_train,y_train[:,i])
File "c:\Users\mat\OneDrive\Desktop\TES\Analisi\Tesi.py", line 72, in mlpottimizzata
mlp_random3.fit(x_train, y_train)
File "C:\Users\mat\AppData\Local\Programs\Python\Python36-32\lib\site-packages\sklearn\model_selection\_search.py", line 639, in fit
cv.split(X, y, groups)))
File "C:\Users\mat\AppData\Local\Programs\Python\Python36-32\lib\site-packages\sklearn\externals\joblib\parallel.py", line 789, in __call__
self.retrieve()
File "C:\Users\mat\AppData\Local\Programs\Python\Python36-32\lib\site-packages\sklearn\externals\joblib\parallel.py", line 740, in retrieve
raise exception
sklearn.externals.joblib.my_exceptions.JoblibValueError: JoblibValueError
___________________________________________________________________________
Multiprocessing exception:
...........................................................................
c:\Users\mat\.vscode\extensions\ms-python.python-2018.6.0\pythonFiles\PythonTools\visualstudio_py_launcher.py in <module>()
86 del sys, os
87
88 # and start debugging
89 ## Begin modification by Don Jayamanne
90 # Pass current Process id to pass back to debugger
---> 91 vspd.debug(filename, port_num, debug_id, debug_options, currentPid, run_as)
92 ## End Modification by Don Jayamanne
...........................................................................
c:\Users\mat\.vscode\extensions\ms-python.python-2018.6.0\pythonFiles\PythonTools\visualstudio_py_debugger.py in debug(file=r'c:\Users\mat\OneDrive\Desktop\TES\Analisi\Tesi.py', port_num=58990, debug_id='34806ad9-833a-4524-8cd6-18ca4aa74f14', debug_options={'RedirectOutput'}, currentPid=10548, run_as='script')
2620 if run_as == 'module':
2621 exec_module(file, globals_obj)
2622 elif run_as == 'code':
2623 exec_code(file, '<string>', globals_obj)
2624 else:
-> 2625 exec_file(file, globals_obj)
file = r'c:\Users\mat\OneDrive\Desktop\TES\Analisi\Tesi.py'
globals_obj = {'__name__': '__main__'}
2626 finally:
2627 sys.settrace(None)
2628 THREADS_LOCK.acquire()
2629 del THREADS[cur_thread.id]
...........................................................................
c:\Users\mat\.vscode\extensions\ms-python.python-2018.6.0\pythonFiles\PythonTools\visualstudio_py_util.py in exec_file(file=r'c:\Users\mat\OneDrive\Desktop\TES\Analisi\Tesi.py', global_variables={'__name__': '__main__'})
114 f = open(file, "rb")
115 try:
116 code = f.read().replace(to_bytes('\r\n'), to_bytes('\n')) + to_bytes('\n')
117 finally:
118 f.close()
--> 119 exec_code(code, file, global_variables)
code = b'import pandas as p\nimport numpy as np\nimport....score(x_train, y_train[:,i]))\n print(err)\n'
file = r'c:\Users\mat\OneDrive\Desktop\TES\Analisi\Tesi.py'
global_variables = {'__name__': '__main__'}
120
121 def exec_module(module, global_variables):
122 '''Executes the provided module as if it were provided as '-m module'. The
123 functionality is implemented using `runpy.run_module`, which was added in
...........................................................................
c:\Users\mat\.vscode\extensions\ms-python.python-2018.6.0\pythonFiles\PythonTools\visualstudio_py_util.py in exec_code(code=b'import pandas as p\nimport numpy as np\nimport....score(x_train, y_train[:,i]))\n print(err)\n', file=r'c:\Users\mat\OneDrive\Desktop\TES\Analisi\Tesi.py', global_variables={'MLPRegressor': <class 'sklearn.neural_network.multilayer_perceptron.MLPRegressor'>, 'RandomForestRegressor': <class 'sklearn.ensemble.forest.RandomForestRegressor'>, 'RandomizedSearchCV': <class 'sklearn.model_selection._search.RandomizedSearchCV'>, '__builtins__': {'ArithmeticError': <class 'ArithmeticError'>, 'AssertionError': <class 'AssertionError'>, 'AttributeError': <class 'AttributeError'>, 'BaseException': <class 'BaseException'>, 'BlockingIOError': <class 'BlockingIOError'>, 'BrokenPipeError':
<class 'BrokenPipeError'>, 'BufferError': <class 'BufferError'>, 'BytesWarning': <class 'BytesWarning'>, 'ChildProcessError': <class 'ChildProcessError'>, 'ConnectionAbortedError': <class 'ConnectionAbortedError'>, ...}, '__cached__': None, '__doc__': None, '__file__': r'c:\Users\mat\OneDrive\Desktop\TES\Analisi\Tesi.py', '__loader__': None, '__name__': '__main__', '__package__': None, ...})
90 if os.path.isdir(sys.path[0]):
91 sys.path.insert(0, os.path.split(file)[0])
92 else:
93 sys.path[0] = os.path.split(file)[0]
94 code_obj = compile(code, file, 'exec')
---> 95 exec(code_obj, global_variables)
code_obj = <code object <module> at 0x02BC45F8, file "c:\Us...at\OneDrive\Desktop\TES\Analisi\Tesi.py", line 1>
global_variables = {'MLPRegressor': <class 'sklearn.neural_network.multilayer_perceptron.MLPRegressor'>, 'RandomForestRegressor': <class 'sklearn.ensemble.forest.RandomForestRegressor'>, 'RandomizedSearchCV': <class 'sklearn.model_selection._search.RandomizedSearchCV'>, '__builtins__': {'ArithmeticError': <class 'ArithmeticError'>, 'AssertionError': <class 'AssertionError'>, 'AttributeError': <class 'AttributeError'>, 'BaseException': <class 'BaseException'>, 'BlockingIOError': <class 'BlockingIOError'>, 'BrokenPipeError': <class 'BrokenPipeError'>, 'BufferError': <class 'BufferError'>, 'BytesWarning': <class 'BytesWarning'>, 'ChildProcessError': <class 'ChildProcessError'>, 'ConnectionAbortedError': <class 'ConnectionAbortedError'>, ...}, '__cached__': None, '__doc__': None, '__file__': r'c:\Users\mat\OneDrive\Desktop\TES\Analisi\Tesi.py', '__loader__': None, '__name__': '__main__', '__package__': None, ...}
96
97 def exec_file(file, global_variables):
98 '''Executes the provided script as if it were the original script provided
99 to python.exe. The functionality is similar to `runpy.run_path`, which was
...........................................................................
c:\Users\mat\OneDrive\Desktop\TES\Analisi\Tesi.py in <module>()
159 # print("Mean squared error: {}".format(rndf_err))
160 # print('Variance score: %.2f \n \n' % rndf.fit(x_train, y_train[:,i]).score(x_test, y_test[:,i]))
161
162 #multilayer perceptron
163 print("Multilayer Perceptron \n")
--> 164 perc = mlpottimizzata(x_train,y_train[:,i])
165 y_perc = perc.predict(x_test)
166 perc_err = mean_squared_error(y_test[:,i], y_perc)
167 err[2,i]=r2_score(y_test[:,i],y_perc)
168 print("Mean squared error: {}".format(perc_err))
...........................................................................
c:\Users\mat\OneDrive\Desktop\TES\Analisi\Tesi.py in mlpottimizzata(x_train=array([[ 0.06 , 2.13 , 4.47
, .... 0.00125208,
0.00505016, 0.0039683 ]]), y_train=array([0.00827529, 0.00318743, 0.00103558, ..., 0.00064697, 0. ,
0.00333603]))
67 activation = ['logistic', 'tanh', 'relu']
68 learning_rate = ['constant', 'invscaling','adaptive']
69 learning_rate_init = [float(x) for x in np.logspace(start = -5, stop = 0, num = 20)]
70 random_grid3 = {'learning_rate': learning_rate,'activation': activation,'learning_rate_init': learning_rate_init, 'hidden_layer_sizes': hidden_layer_sizes, 'alpha': alpha}
71 mlp_random3 = RandomizedSearchCV(estimator = mlpsgd, param_distributions = random_grid3, n_iter = 350, n_jobs=-1)
---> 72 mlp_random3.fit(x_train, y_train)
mlp_random3.fit = <bound method BaseSearchCV.fit of RandomizedSear...urn_train_score='warn', scoring=None, verbose=0)>
x_train = array([[ 0.06 , 2.13 , 4.47 , .... 0.00125208,
0.00505016, 0.0039683 ]])
y_train = array([0.00827529, 0.00318743, 0.00103558, ..., 0.00064697, 0. ,
0.00333603])
73
74 if mlp_random3.best_score_ is max(mlp_random1.best_score_,mlp_random2.best_score_,mlp_random3.best_score_):
75 return mlp_random3.best_estimator_
76 if mlp_random1.best_score_ >= mlp_random2.best_score_:
...........................................................................
C:\Users\mat\AppData\Local\Programs\Python\Python36-32\lib\site-packages\sklearn\model_selection\_search.py in fit(self=RandomizedSearchCV(cv=None, error_score='raise',...turn_train_score='warn', scoring=None, verbose=0), X=array([[ 0.06 , 2.13 , 4.47 , .... 0.00125208,
0.00505016, 0.0039683 ]]), y=array([0.00827529, 0.00318743, 0.00103558, ..., 0.00064697, 0. ,
0.00333603]), groups=None, **fit_params={})
634 return_train_score=self.return_train_score,
635 return_n_test_samples=True,
636 return_times=True, return_parameters=False,
637 error_score=self.error_score)
638 for parameters, (train, test) in product(candidate_params,
--> 639 cv.split(X, y, groups)))
cv.split = <bound method _BaseKFold.split of KFold(n_splits=3, random_state=None, shuffle=False)>
X = array([[ 0.06 , 2.13 , 4.47 , .... 0.00125208,
0.00505016, 0.0039683 ]])
y = array([0.00827529, 0.00318743, 0.00103558, ..., 0.00064697, 0. ,
0.00333603])
groups = None
640
641 # if one choose to see train score, "out" will contain train score info
642 if self.return_train_score:
643 (train_score_dicts, test_score_dicts, test_sample_counts, fit_time,
...........................................................................
C:\Users\mat\AppData\Local\Programs\Python\Python36-32\lib\site-packages\sklearn\externals\joblib\parallel.py in __call__(self=Parallel(n_jobs=-1), iterable=<generator object BaseSearchCV.fit.<locals>.<genexpr>>)
784 if pre_dispatch == "all" or n_jobs == 1:
785 # The iterable was consumed all at once by the above for loop.
786 # No need to wait for async callbacks to trigger to
787 # consumption.
788 self._iterating = False
--> 789 self.retrieve()
self.retrieve = <bound method Parallel.retrieve of Parallel(n_jobs=-1)>
790 # Make sure that we get a last message telling us we are done
791 elapsed_time = time.time() - self._start_time
792 self._print('Done %3i out of %3i | elapsed: %s finished',
793 (len(self._output), len(self._output),
---------------------------------------------------------------------------
Sub-process traceback:
---------------------------------------------------------------------------
ValueError Tue Jul 17 19:33:23 2018
PID: 9280Python 3.6.5: C:\Users\mat\AppData\Local\Programs\Python\Python36-32\python.exe
...........................................................................
C:\Users\mat\AppData\Local\Programs\Python\Python36-32\lib\site-packages\sklearn\externals\joblib\parallel.py in __call__(self=<sklearn.externals.joblib.parallel.BatchedCalls object>)
126 def __init__(self, iterator_slice):
127 self.items = list(iterator_slice)
128 self._size = len(self.items)
129
130 def __call__(self):
--> 131 return [func(*args, **kwargs) for func, args, kwargs in self.items]
self.items = [(<function _fit_and_score>, (MLPRegressor(activation='relu', alpha=811.130830...tion=0.1,
verbose=False, warm_start=False), array([[ 6.00000000e-02, 2.13000000e+00, 4.470...25207638e-03, 5.05016074e-03, 3.96830145e-03]]), array([0.00827529, 0.00318743, 0.00103558, ..., 0.00064697, 0. ,
0.00333603]), {'score': <function _passthrough_scorer>}, array([ 629, 630, 631, ..., 1882, 1883, 1884]), array([ 0, 1, 2, 3, 4, 5, 6, 7, ..., 621, 622, 623,
624, 625, 626, 627, 628]), 0, {'activation': 'relu', 'alpha': 811.130830789689, 'hidden_layer_sizes': (24,),
'learning_rate': 'adaptive', 'learning_rate_init': 0.5455594781168515}), {'error_score': 'raise', 'fit_params': {},
'return_n_test_samples': True, 'return_parameters': False, 'return_times': True, 'return_train_score': 'warn'})]
132
133 def __len__(self):
134 return self._size
135
...........................................................................
C:\Users\mat\AppData\Local\Programs\Python\Python36-32\lib\site-packages\sklearn\externals\joblib\parallel.py in <listcomp>(.0=<list_iterator object>)
126 def __init__(self, iterator_slice):
127 self.items = list(iterat
There is nothing missing, it ends like this. Also I need to mention that mplottimizzata, which is cited in the error, is the function that contains the first block of code I attached. I am really out of options, any help is really appreciated. Thank you all in advance :)
NB. Another part of the code does roughly the same thing but with solver:'lbfgs' and it works smoothly, but this only confuses me even further.
The problem is caused when you define the grid parameters using list comprehension and the float argument.
This works fine for me:
from sklearn.neural_network import MLPRegressor
from sklearn.model_selection import RandomizedSearchCV
import pandas as pd
import numpy as np
from sklearn.model_selection import GridSearchCV
X = pd.read_csv('X.csv')
Y = pd.read_csv('y.csv')
X = X.iloc[1:,1:].values
Y = Y.iloc[1:,1].values
mlpsgd = MLPRegressor(max_iter = 1000, solver='sgd')
alpha = np.arange(0.01, 0.1, 0.01)
hidden_layer_sizes = [(int(x),int(y),int(z)) for x in np.logspace(start = 0, stop = 2.2, num = 8) for y in np.logspace(start = 0, stop = 2.2, num = 8) for z in np.logspace(start = 0, stop = 2.2, num = 8)]
hidden_layer_sizes.extend((int(x),int(y)) for x in np.logspace(start = 0, stop = 2, num = 25) for y in np.logspace(start = 0, stop = 2, num = 25))
hidden_layer_sizes.extend((int(x),) for x in np.logspace(start = 1, stop = 2, num = 1000))
activation = ['logistic', 'tanh', 'relu']
learning_rate = ['constant', 'invscaling','adaptive']
learning_rate_init = np.arange(0.01, 0.1, 0.01)
random_grid3 = {'learning_rate': learning_rate,'activation': activation,'learning_rate_init': learning_rate_init, 'hidden_layer_sizes': hidden_layer_sizes, 'alpha': alpha}
mlp_random3 = RandomizedSearchCV(estimator = mlpsgd, param_distributions = random_grid3, n_iter = 350, n_jobs=-1)
mlp_random3.fit(X, Y)
print(mlp_random3.best_estimator_)
MLPRegressor(activation='relu', alpha=0.03, batch_size='auto', beta_1=0.9, beta_2=0.999, early_stopping=False, epsilon=1e-08, hidden_layer_sizes=(4, 18, 1), learning_rate='adaptive', learning_rate_init=0.05, max_iter=1000, momentum=0.9, nesterovs_momentum=True, power_t=0.5, random_state=None, shuffle=True, solver='sgd', tol=0.0001, validation_fraction=0.1, verbose=False, warm_start=False)